                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                   O) processors and in time log? n. 


EXAMPLE 6.19 
Given two n bit numbers x,y we wish to compute «+ y fast in parallel. The gradeschool algorithm 
proceeds from the least significant bit and maintains a carry bit. The most significant bit is 
computed only after n steps. This algorithm does not take advantage of parallelism. A better 
algorithm called carry lookahead assigns each bit position to a separate processor and then uses 
interprocessor communication to propagate carry bits. It takes O(n) processors and O(log n) time. 
There are also efficient parallel algorithms for integer multiplication and division (the latter is 
quite nonintuitive and unlike the gradeschool algorithm!). 


EXAMPLE 6.20 
Many matrix computations can be done efficiently in parallel: these include computing the product, 
rank, determinant, inverse, etc. (See exercises.) 

Some graph theoretic algorithms such as shortest paths and minimum spanning tree also have 
fast parallel implementations. 

But many well-known polynomial-time problems such as minimum matching, maximum flows, 
and linear programming are not known to have any good parallel implementations and are conjec- 
tured not to have any; see our discussion of P-completeness below. 


Now we relate parallel computation to circuits. The depth of a circuit is the length of the longest 
directed path from an input node to the output node. 


DEFINITION 6.21 (NICK’S CLASS OR NC) 

A language is in NC’ if there are constants c,d > 0 such that it can be decided by a logspace- 
uniform family of circuits {C} where C, has size O(n°) and depth O(log*n). The class NC is 
Uis1NCt. 


A related class is the following. 


DEFINITION 6.22 (AC) l 
The class AC” is defined similarly to NC’ except gates are allowed to have unbounded fanin. The 
class AC is Ujs9 AC’. 
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Since unbounded (but poly(n)) fanin can be simulated using a tree of ORs/ANDs of depth 
O(log n), we have NC’ C AC? C NC'*!, and the inclusion is known to be strict for i = 0 as we 
will see in Chapter 13. (Notice, NC? is extremely limited since the circuit's output depends upon 
a constant number of input bits, but AC? does not suffer from this limitation.) 


EXAMPLE 6.23 

The language PARITY ={a: x has an odd number of 1s} is in NC!. The circuit computing it 
has the form of a binary tree. The answer appears at the root; the left subtree computes the parity 
of the first |x| /2 bits and the right subtree computes the parity of the remaining bits. The gate 
at the top computes the parity of these two bits. Clearly, unwrapping the recursion implicit in our 
description gives a circuit of dept O(log n). 


The classes AC, NC are important because of the following. 


‘THEOREM 6.24 
A language has efficient parallel algorithms iff it is in NC. 


PROOF: Suppose a language L € NC and is decidable by a circuit family {Cn} where Cn has size 
N = O(n°) and depth D = O(log’n). Take a general purpose parallel computer with N nodes 
and configure it to decide L as follows. Compute a description of C,, and allocate the role of each 
circuit node to a distinct processor. (This is done once, and then the computer is ready to compute 
on any input of length n.) Each processor, after computing the output at its assigned node, sends 
the resulting bit to every other circuit node that needs it. Assuming the interconnection network 
delivers all messages in O(log N) time, the total running time is O(log’! N). 

The reverse direction is similar, with the circuit having N - D nodes arranged in D layers, and 
the ith node in the tth layer performs the computation of processor ¿ at time t. The role of the 
interconnection network is played by the circuit wires. M 


6.5.2 P-completeness 


A major open question in this area is whether P = NC. We believe that the answer is NO 
(though we are currently even unable to separate PH from NC”). This motivates the theory of 
P-completeness, a study of which problems are likely to be in NC and which are not. 


DEFINITION 6.25 
A language is P-complete if it is in P and every language in P is logspace-reducible to it (as per 


Definition 4.14). 
The following easy theorem is left for the reader as Exercise 12. 


‘THEOREM 6.26 
If language L is P-complete then 


1. L€ NC if P = NC. 
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2. L € L if P = L. (Where L is the set languages decidable in logarithmic space, see Defini- 
tion 4.5.) 


The following is a fairly natural P-complete language: 


THEOREM 6.27 
Let CIRCUIT-EVAL denote the language consisting of all pairs (C,x) such that C is an n-inputs 
single-output circuit and x € {0,1}" satisfies C(x) = 1. Then CIRCUIT-EVAL is P-complete. 


PROOF: The language is clearly in P. A logspace-reduction from any other language in P to this 
language is implicit in the proof of Theorem 6.7. W 


6.6 Circuits of exponential size 


As noted, every language has circuits of size O(n2"). However, actually finding these circuits may 
be difficult— sometimes even undecidable. If we place a uniformity condition on the circuits, that 
is, require them to be efficiently computable then the circuit complexity of some languages could 
exceed n2”. In fact it is possible to give alternative definitions of some familiar complexity classes, 
analogous to the definition of P in Theorem 6.7. 


DEFINITION 6.28 (DC-UNIFORM) 

Let (Cr }n>1 be a circuit family. We say that it is a Direct Connect uniform (DC uniform) family if, 
given (n, i), we can compute in polynomial time the it? but of (the representation of) the circuit Cp. 
More concretely, we use the adjacency matrix representation and hence a family (Cn nen is DC 
uniform iff the functions SIZE, TYPE and EDGE defined in Remark ?? are computable in polynomial 
time. 


Note that the circuits may have exponential size, but they have a succinct representation in 
terms of a TM which can systematically generate any required node of the circuit in polynomial 
time. 

Now we give a (yet another) characterization of the class PH, this time as languages computable 
by uniform circuit families of bounded depth. We leave it as Exercise 13. 


THEOREM 6.29 
Le PH iff L can be computed by a DC uniform circuit family {Cn} that 


e uses AND, OR, NOT gates. 
e has size 2" and constant depth (i.e., depth O(1)). 
e gates can have unbounded (exponential) fanin. 


e the NOT gates appear only at the input level. 


If we drop the restriction that the circuits have constant depth, then we obtain exactly EXP 
(see Exercise 14). 
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6.7 Circuit Satisfiability and an alternative proof of the Cook- 
Levin Theorem 


Boolean circuits can be used to define the following NP-complete language: 


DEFINITION 6.30 

The circuit satisfiability language CKT-SAT consists of all (strings representing) circuits with a 
single output that have a satisfying assignment. That is, a string representing an n-input circuit 
C is in CKT-SAT iff there exists u € {0,1}” such that C(u) = 1. 


CKT-SAT is clearly in NP because the satisfying assignment can serve as the certificate. It is 
also clearly NP-hard as every CNF formula is in particular a Boolean circuit. However, CKT-SAT 
can also be used to give an alternative proof (or, more accurately, a different presentation of the 
same proof) for the Cook-Levin Theorem by combining the following two lemmas: 


LEMMA 6.31 
CKT-SAT is NP-hard. 


PROOF: Let L be an NP-language and let p be a polynomial and M a polynomial-time TM such 
that x € L iff M(x,u) = 1 for some u € {0,1}?"")), We reduce L to CKT-SAT by mapping (in 
polynomial-time) z to a circuit Cy with p(|x|) inputs and a single output such that C,(u) = M(z, u) 
for every u € Lo, 10D, Clearly, x € L & C, € CKT-SAT and so this suffices to show that 
L <p CKT-SAT. 

Yet, it is not hard to come up with such a circuit. Indeed, the proof of Theorem 6.7 yields a 
way to map M, x into the circuit C, in logarithmic space (which in particular implies polynomial 
time). Ml 


LEMMA 6.32 
CKT-SAT <, 3SAT 


PROOF: As mentioned above this follows from the Cook-Levin theorem but we give here a direct 
reduction. If C is a circuit, we map it into a 3CNF formula ọ as follows: 

For every node v; of C we will have a corresponding variable z; in y. If the node v; is an AND of 
the nodes vj and vz then we add to y clauses that are equivalent to the condition “z; = (zj A Zk)”. 
That is, we add the clauses 


Zi V Zi V 2p) AN (25 V Zi V Zp) A (25 V Zi VW Ze) A (2; V 23 V Zk) 
I J I J 


Similarly, if v; is an OR of vj and vz then we add clauses equivalent to “z; = (zj V 24)”, and if vi 
is the NOT of v; then we add the clauses (z; V z;) A (Z; V Zj). 

Finally, if v; is the output node of C then we add the clause z; to y. It is not hard to see that 
the formula y will be satisfiable if and only if the circuit C is. W 
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WHAT HAVE WE LEARNED? 


e Boolean circuits can be used as an alternative computational model to TMs. 
The class P /poly of languages decidable by polynomial-sized circuits is a strict 
superset of P but does not contain NP unless the hierarchy collapses. 


e Almost every function from (0, 1)” to {0, 1} requires exponential-sized circuits. 
Finding even one function in NP with this property would show that P 4 NP. 


e The class NC of languages decidable by (uniformly constructible) circuits with 
polylogarithmic depth and polynomial size corresponds to computational tasks 
that can be efficiently parallelized. 


Chapter notes and history 


Karp-Lipton theorem is from [KL82]. Karp and Lipton also gave a more general definition of advice 
that can be used to define the class C/a(n) for every complexity class C and function a. However, 
we do not use this definition here since it does not seem to capture the intuitive notion of advice 
for classes such as NP N coNP, BPP and others. 

The class of NC algorithms as well as many related issues in parallel computation are discussed 
in Leighton [?]. 


Exercises 


§1 [Kannan [Kan82]] Show for every k > 0 that PH contains languages whose circuit complexity 
is Q(n*). 


"Ayixo]dunoo pnn ysy 
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$2 Solve the previous question with PH replaced by ¥5. 
$3 ([KL82], attributed to A. Meyer) Show that if EXP C P/poly then EXP = X}. 
§4 Show that if P = NP then there is a language in EXP that requires circuits of size 2”/n. 


§5 A language L C {0,1}" is sparse if there is a polynomial p such that |L N {0,1}"| < p(n) for 
every n € N. Show that every sparse language is in P/poly. 


86 (X’s Theorem 19??) Show that if a sparse language is NP-complete then P = NP. (This is 
a strengthening of Exercise 13 of Chapter 2.) 
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87 Show a logspace implicitly computable function f that maps any n-vertex graph in adjacency 
matrix representation into the same graph in adjacency list representation. You can think 
of the adjacency list representation of an n-vertex graph as a sequence of n strings of size 
O(nlog n) each, where the i” string contains the list of neighbors of the it” vertex in the 
graph (and is padded with zeros if necessary). 
§8 (Open) Suppose we make a stronger assumption than NP C P/poly: every language in NP 
has linear size circuits. Can we show something stronger than PH = x5? 
89 (a) Describe an NC circuit for the problem of computing the product of two given n x n 
matrices A, B. 
(b) Describe an NC circuit for computing, given an n x n matrix, the matrix A”. 
‘e(,_,zV) = „zV Burrenbs poyeodaz os) PUH 
(c) Conclude that the PATH problem (and hence every NL language) is in NC. 
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$10 A formula is a circuit in which every node (except the input nodes) has outdegree 1. Show 
that a language is computable by polynomial-size formulae iff it is in (nonuniform) NC!. 
‘yore g/g sow 
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$11 Show that NC! = L. Conclude that PSPACE 4 NC’. 
§12 Prove Theorem 6.26. That is, prove that if L is P-complete then L € NC (resp. L) iff 
P = NC (resp. L). 
§13 Prove Theorem 6.29 (that PH is the set of languages with constant-depth DC uniform cir- 
cuits). 
$14 Show that EXP is exactly the set of languages with DC uniform circuits of size 2” where c 
is some constant (c may depend upon the language). 
$15 Show that if linear programming has a fast parallel algorithm then P = NC. 
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Chapter 7 


Randomized Computation 


“We do not assume anything about the distribution of the instances of the problem 
to be solved. Instead we incorporate randomization into the algorithm itself... It may 
seem at first surprising that employing randomization leads to efficient algorithm. 
This claim is substantiated by two examples. The first has to do with finding the 
nearest pair in a set of n points in RE. The second example is an extremely efficient 
algorithm for determining whether a number is prime.” 

Michael Rabin, 1976 


Thus far our standard model of computation has been the deterministic Turing Machine. But 
everybody who is even a little familiar with computation knows that that real-life computers need 
not be deterministic since they have built-in ”random number generators.” In fact these generators 
are very useful for computer simulation of ”random” processes such as nuclear fission or molecular 
motion in gases or the stock market. This chapter formally studies probablistic computation, and 
complexity classes associated with it. 

We should mention right away that it is an open question whether or not the universe has any 
randomness in it (though quantum mechanics seems to guarantee that it does). Indeed, the output 
of current "random number generators” is not guaranteed to be truly random, and we will revisit 
this limitation in Section 7.4.3. For now, assume that true random number generators exist. Then 
arguably, a realistic model for a real-life computer is a Turing machine with a random number 
generator, which we call a Probabilistic Turing Machine (PTM). It is natural to wonder whether 
difficult problems like 3SAT are efficiently solvable using a PTM. 

We will formally define the class BPP of languages decidable by polynomial-time PTMs and 
discuss its relation to previously studied classes such as P/poly and PH. One consequence is that 
if PH does not collapse, then 3SAT does not have efficient probabilistic algorithms. 

We also show that probabilistic algorithms can be very practical by presenting ways to greatly 
reduce their error to absolutely minuscule quantities. Thus the class BPP (and its sister classes 
RP,coRP and ZPP) introduced in this chapter are arguably as important as P in capturing 
efficient computation. We will also introduce some related notions such as probabilistic logspace 
algorithms and probabilistic reductions. 


p7.1 (115) 
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Though at first randomization seems merely a tool to allow simulations of randomized physical 
processes, the surprising fact is that in the past three decades randomization has led to more efficient 
—and often simpler—algorithms for problems in a host of other fields—such as combinatorial 
optimization, algebraic computation, machine learning, and network routing. 

In complexity theory too, the role of randomness extends far beyond a study of randomized 
algorithms and classes such as BPP. Entire areas such as cryptography and interactive and prob- 
abilistically checkable proofs rely on randomness in an essential way, sometimes to prove results 
whose statement did not call for randomness at all. The groundwork for studying those areas will 
be laid in this chapter. 

In a later chapter, we will learn something intriguing: to some extent, the power of randomness 
may be a mirage. If a certain plausible complexity-theoretic conjecture is true (see Chapters 16 
and 17), then every probabilistic algorithm can be simulated by a deterministic algorithm (one that 
does not use any randomness whatsoever) with only polynomial overhead. 

Throughout this chapter and the rest of the book, we will use some notions from elementary 
probability on finite sample spaces; see Appendix A for a quick review. 


7.1 Probabilistic Turing machines 


We now define probabilistic Turing machines (PTMs). Syntactically, a PTM is no different from a 
nondeterministic TM: it is a TM with two transition functions 69, ô1. The difference lies in how we 
interpret the graph of all possible computations: instead of asking whether there exists a sequence 
of choices that makes the TM accept, we ask how large is the fraction of choices for which this 
happens. More precisely, if M is a PTM, then we envision that in every step in the computation, 
M chooses randomly which one of its transition functions to apply (with probability half applying 
ôo and with probability half applying 91). We say that M decides a language if it outputs the right 
answer with probability at least 2/3. 

Notice, the ability to pick (with equal probability) one of ĉo, 61 to apply at each step is equivalent 
to the machine having a ”fair coin”, which, each time it is tossed, comes up ”Heads” or ” Tails” 
with equal probability regardless of the past history of Heads/Tails. As mentioned, whether or not 
such a coin exists is a deep philosophical (or scientific) question. 


DEFINITION 7.1 (THE CLASSES BPTIME AND BPP) 

For T : N > N and LC {0,1}", we say that a PTM M decides L in time T(n), if 
for every x € {0,1}", M halts in 7 (|x|) steps regardless of its random choices, and 
Pr[ M(x) = L(x)] > 2/3, where we denote L(x) = 1 if x € Land L(x) =0if x ¢ L. 
We let BPTIME(T(n)) denote the class of languages decided by PTMs in O(T(n)) 
time and let BPP = U-BPTIME(n‘). 


REMARK 7.2 
We will see in Section 7.4 that this definition is quite robust. For instance, the ”coin” need not 
be fair. The constant 2/3 is arbitrary in the sense that it can be replaced with any other constant 
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greater than half without changing the classes BPTIME(T(n)) and BPP. Instead of requiring 
the machine to always halt in polynomial time, we could allow it to halt in expected polynomial 
time. 


REMARK 7.3 

While Definition 7.1 allows the PTM M, given input x, to output a value different from L(x) 
with positive probability, this probability is only over the random choices that M makes in the 
computation. In particular for every input x, M(x) will output the right value L(x) with probability 
at least 2/3. Thus BPP, like P, is still a class capturing worst-case computation. 


Since a deterministic TM is a special case of a PTM (where both transition functions are equal), 
the class BPP clearly contains P. As alluded above, under plausible complexity assumptions it 
holds that BPP = P. Nonetheless, as far as we know it may even be that BPP = EXP. (Note 
that BPP C EXP, since given a polynomial-time PTM M and input x € {0,1}” in time gpoly(n) 
it is possible to enumerate all possible random choices and compute precisely the probability that 
M(a) =1.) 


An alternative definition. As we did with NP, we can define BPP using deterministic TMs 
where the ” probabilistic choices” to apply at each step can be provided to the TM as an additional 
input: 


DEFINITION 7.4 (BPP, ALTERNATIVE DEFINITION) 
BPP contains a language L if there exists a polynomial-time TM M and a polynomial p: N > N 


such that for every x € {0,1}*, ento pede [M (z, r) = La] > 2/3. 


7.2 Some examples of PTMs 


The following examples demonstrate how randomness can be a useful tool in computation. We will 
see many more examples in the rest of this book. 


7.2.1 Probabilistic Primality Testing 


In primality testing we are given an integer N and wish to determine whether or not it is prime. 
Generations of mathematicians have learnt about prime numbers and —before the advent of 
computers— needed to do primality testing to test various conjectures’. Ideally, we want effi- 
cient algorithms, which run in time polynomial in the size of N’s representation, in other words, 
poly(logn). We knew of no such efficient algorithms? until the 1970s, when an effficient proba- 
bilistic algorithm was discovered. This was one of the first to demonstrate the power of proba- 
bilistic algorithms. In a recent breakthrough, Agrawal, Kayal and Saxena [?] gave a deterministic 
polynomial-time algorithm for primality testing. 


Though a very fast human computer himself, Gauss used the help of a human supercomputer —an autistic person 
who excelled at fast calculations—to do primality testing. 

2In fact, in his letter to von Neumann quoted in Chapter 2, Gödel explicitly mentioned this problem as an example 
for an interesting problem in NP but not known to be efficiently solvable. 
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Formally, primality testing consists of checking membership in the language PRIMES = { _N,: N is a prime nun 
Notice, the corresponding search problem of finding the factorization of a given composite number 
N seems very different and much more difficult. It is the famous FACTORING problem, whose 
conjectured hardness underlies many current cryptosystems. Chapter 20 describes Shor’s algorithm 
to factors integers in polynomial time in the model of quantum computers. 
We sketch an algorithm showing that PRIMES is in BPP (and in fact in coRP). For every 
number N, and A € [N — 1], define 


O gcd(A,N) #1 
A is a quadratic residue modulo N 
That is, A= B? (mod N) for some B with gcd(B, N) = 1 


—1 otherwise 


We use the following facts that can be proven using elementary number theory: 
e For every odd prime N and A € [N — 1], QRn(A) = AW=D2 (mod N). 


e For every odd N, A define the Jacobi symbol (4) as Tey QRp,(A) where P;,..., Pz are all 


the (not necessarily distinct) prime factors of N (i.e., N = ME P;). Then, the Jacobi symbol 
is computable in time O(log A - log N). 


e For every odd composite N, |{A € [N — 1] : gcd(N, A)=1 and (4) = AN-D/2)| < 5|{A € 
[N — 1] : ged(N, A) = 1}| 


Together these facts imply a simple algorithm for testing primality of N (which we can assume 
without loss of generality is odd): choose a random 1 < A < N, if ged(N, A) > 1 or (X) 2 AW-1)/2 
(mod N) then output “composite”, otherwise output “prime”. This algorithm will always output 
“prime” is N is prime, but if N is composite will output “composite” with probability at least 1/2. 
(Of course this probability can be amplified by repeating the test a constant number of times.) 


7.2.2 Polynomial identity testing 


So far we described probabilistic algorithms solving problems that have known deterministic poly- 
nomial time algorithms. We now describe a problem for which no such deterministic algorithm is 
known: 

We are given a polynomial with integer coefficients in an implicit form, and we want to decide 
whether this polynomial is in fact identically zero. We will assume we get the polynomial in the 
form of an arithmetic circuit. This is analogous to the notion of a Boolean circuit, but instead of the 
operators A, V and ~, we have the operators +,— and x. Formally, an n-variable arithmetic circuit 
is a directed acyclic graph with the sources labeled by a variable name from the set 21,...,Tp, 
and each non-source node has in-degree two and is labeled by an operator from the set {+,—, x}. 
There is a single sink in the graph which we call the output node. The arithmetic circuit defines 
a polynomial from Z” to Z by placing the inputs on the sources and computing the value of 
each node using the appropriate operator. We define ZEROP to be the set of arithmetic circuits 
that compute the identically zero polynomial. Determining membership in ZEROP is also called 


7.2. SOME EXAMPLES OF PTMS p7.5 (119) 


polynomial identity testing, since we can reduce the problem of deciding whether two circuits C, C” 
compute the same polynomial to ZEROP by constructing the circuit D such that D(z1,..., £n) = 
O (tipes ie) — O (tipyn): 

Since expanding all the terms of a given arithmetic circuit can result in a polynomial with 
exponentially many monomials, it seems hard to decide membership in ZEROP. Surprisingly, there 
is in fact a simple and efficient probabilistic algorithm for testing membership in ZEROP. At the 
heart of this algorithm is the following fact, typically known as the Schwartz-Zippel Lemma, whose 
proof appears in Appendix A (see Lemma A.25): 


LEMMA 7.5 
Let p(11,t2,...,%m) be a polynomial of total degree at most d and S is any finite set of integers. 
When a1,a2,..., dm are randomly chosen with replacement from S, then 
d 
Pr|p(ai,a2,...,a@m) 40] > 1— is 


Now it is not hard to see that given a size m circuit C on n variables, it defines a polynomial of 
degree at most 2”. This suggests the following simple probabilistic algorithm: choose n numbers 
£1,.-.,%p from 1 to 10-2” (this requires O(n-m) random bits), evaluate the circuit C on 21,...,Tn 
to obtain an output y and then accept if y = 0, and reject otherwise. Clearly if C € ZEROP then 
we always accept. By the lemma, if C ¢ ZEROP then we will reject with probability at least 9/10. 

However, there is a problem with this algorithm. Since the degree of the polynomial represented 
by the circuit can be as high as 2”, the output y and other intermediate values arising in the 
computation may be as large as (10- 2)?” — this is a value that requires exponentially many bits 
just to describe! 

We solve this problem using the technique of fingerprinting. The idea is to perform the evalu- 
ation of C on 21,...,%y modulo a number k that is chosen at random in [22]. Thus, instead of 
computing y = C(x1,..., tn), we compute the value y (mod k). Clearly, if y = 0 then y (mod k) is 
also equal to 0. On the other hand, we claim that if y Æ 0, then with probability at least 6 = com 
k does not divide y. (This will suffice because we can repeat this procedure O(1/0) times to ensure 
that if y 4 0 then we find this out with probability at lest 9/10.) Indeed, assume that y 4 0 and 
let S = {pi,..., pe} denote set of the distinct prime factors of y. It is sufficient to show that with 
probability at ô, the number k will be a prime number not in S. Yet, by the prime number theorem, 
the probability that k is prime is at least = = 26. Also, since y can have at most log y < 5m2™ 
distinct factors, the probability that k is in S is less than ue < aim = 6. Hence by the union 
bound, with probability at least 6, k will not divide y. 


7.2.3 Testing for perfect matching in a bipartite graph. 


If G = (Vi, Va, E) is the bipartite graph where |Vj| = |V2| and E C Vi x Va then a perfect matching 
is some E' C E such that every node appears exactly once among the edges of E’. Alternatively, 
we may think of it as a permutation o on the set {1,2,...,n} (where n = |Vj|) such that for 
each ¿€ {1,2,...,n}, the pair (¿,0(1)) is an edge. Several deterministic algorithms are known for 
detecting if a perfect matching exists. Here we describe a very simple randomized algorithm (due 
to Lovász) using the Schwartz-Zippel lemma. 
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Consider the n x n matrix X (where n = |Vj| = |V2|) whose (i, j) entry Xj; is the variable x;; 
if (i,j) € E and 0 otherwise. Recall that the determinant of matrix det(X) is 


n 


det(X) = y (=1)8e) TEZO (1) 


aESn i=1 


where S,, is the set of all permutations of {1,2,...,n}. Note that every permutation is a potential 
perfect matching, and the corresponding monomial in det(X) is nonzero iff this perfect matching 
exists in G. Thus the graph has a perfect matching iff det(X) 4 0. 

Now observe two things. First, the polynomial in (1) has |E| variables and total degree at most 
n. Second, even though this polynomial may be of exponential size, for every setting of values to 
the X;; variables it can be efficiently evaluated, since computing the determinant of a matrix with 
integer entries is a simple polynomial-time computation (actually, even in NC?). 

This leads us to Lovász's randomized algorithm: pick random values for X;;’s from [1,..., 2n], 
substitute them in X and compute the determinant. If the determinant is nonzero, output “accept” 
else output “reject.” The advantage of the above algorithm over classical algorithms is that it can 
be implemented by a randomized NC circuit, which means (by the ideas of Section 6.5.1) that it 
has a fast implementation on parallel computers. 


7.3 One-sided and zero-sided error: RP, coRP, ZPP 


The class BPP captured what we call probabilistic algorithms with two sided error. That is, it 
allows the machine M to output (with some small probability) both 0 when z € L and 1 when 
x ¢ L. However, many probabilistic algorithms have the property of one sided error. For example 
if x E L they will never output 1, although they may output 0 when x € L. This is captured by 
the definition of RP. 

DEFINITION 7.6 

RTIME(t(n)) contains every language L for which there is a is a probabilistic TM M running in 
t(n) time such that 


2 
x € L= Pr|M accepts z] > 3 
x g L= Pr|M accepts 1] = 0 

We define RP = U¿¿RTIME(nS). 


Note that RP C NP, since every accepting branch is a “certificate” that the input is in the 
language. In contrast, we do not know if BPP C NP. The class coRP = {L | L € RP} captures 
one-sided error algorithms with the error in the “other direction” (i.e., may output 1 when z ¢ L 
but will never output 0 if x € L). 

For a PTM M, and input z, we define the random variable Try, to be the running time of M 
on input x. That is, Pr[Tm, = = T] = p if with probability p over the random choices of M on input 
x, it will halt within T steps. We say that M has expected running time T(n) if the expectation 
E[Tiv.c] is at most T(|x|) for every x € {0,1}". We now define PTMs that never err (also called 
“zero error” machines): 
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DEFINITION 7.7 
The class ZTIME(T(n)) contains all the languages L for which there is an expected-time O(T(n)) 
machine that never errs. That is, 


x € L => Pr|M accepts z] = 1 
x g L= Pr[M halts without accepting on x] = 1 


We define ZPP = Ue>soZ TIME (n°). 


The next theorem ought to be slightly surprising, since the corresponding statement for nonde- 
terminism is open; i.e., whether or not P = NP N coNP. 


THEOREM 7.8 
ZPP = RP A coRP. 


We leave the proof of this theorem to the reader (see Exercise 4). To summarize, we have the 
following relations between the probabilistic complexity classes: 


ZPP =RP NcoRP 
RP CBPP 
coRP CBPP 


7.4 The robustness of our definitions 


When we defined P and NP, we argued that our definitions are robust and were likely to be the 
same for an alien studying the same concepts in a faraway galaxy. Now we address similar issues 
for probabilistic computation. 


7.4.1 Role of precise constants, error reduction. 


The choice of the constant 2/3 seemed pretty arbitrary. We now show that we can replace 2/3 with 
any constant larger than 1/2 and in fact even with 1/2 + n~© for a constant c > 0. 


LEMMA 7.9 
For c > 0, let BPP,,-- denote the class of languages L for which there is a polynomial-time PTM 
M satisfying Pr|M (x) = L(x)] > 1/2 + |x|~¢ for every x € {0,1}*. Then BPP,,-. = BPP. 


Since clearly BPP C BPP,,--, to prove this lemma we need to show that we can transform a 
machine with success probability 1/2-+n~° into a machine with success probability 2/3. We do this 
by proving a much stronger result: we can transform such a machine into a machine with success 
probability exponentially close to one! 
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THEOREM 7.10 (ERROR REDUCTION) 

Let L C {0,1}* be a language and suppose that there exists a polynomial-time PTM 
M such that for every x € {0,1}*, Pr[M(x) = L(x) > 4 + |2|~°. 

Then for every constant d > 0 there exists a polynomial-time PTM M' such that 
for every x € {0,1}*, Pr[M' (x) = L(x)] > 1 —27!#". 


PROOF: The machine M’ is quite simple: for every input x € {0,1}", run M(x) for k times 
obtaining k outputs y,,...,yr € {0,1}, where k = 8|a|?4+°. If the majority of these values are 1 
then accept, otherwise reject. 

To analyze this machine, define for every i € [k] the random variable X; to equal 1 if y; = L(x) 
and to equal 0 otherwise. Note that Xy,..., Xy are independent Boolean random variables with 
E[X;] = Pr[X; = 1] > 1/2 + n° (where n = |x|). The Chernoff bound (see Theorem A.18 in 
Appendix A) implies the following corollary: 

COROLLARY 7.11 
Let X1,..., Xp be independent identically distributed Boolean random variables, with Pr[X; = 
1] = p for every 1 < i < k. Let 6 € (0,1). Then, 


k 
i» Xi-p 
i=1 


In our case p = 1/2+n~°, and plugging in 6 = n~°/2, the probability we output a wrong answer 
is bounded by 


62 
Pr[ >ð] ap TK 


k 
LL 
Pr[4 So Xi < 1/2 at n7/2] < e” nz 58n? +d < gn 
1=1 


A similar result holds for the class RP. In fact, there we can replace the constant 2/3 with 
every positive constant, and even with values as low as n”“. That is, we have the following result: 


‘THEOREM 7.12 
Let L C {0,1}* such that there exists a polynomial-time PTM M satisfying for every x € {0,1}*: 
(1) If x € L then Pr[M(x) =1)] > n~ and (2) if x ¢ L, then Pr[M(x) = 1] =0. 

Then for every d > 0 there exists a polynomial-time PTM M’ such that for every x € {0,1}", 
(1) ifx € L then Pr[M’(x) = 1] >1—2-"" and (2) if x ¢ L then Pr[M’(x) = 1] =0. 


These results imply that we can take a probabilistic algorithm that succeeds with quite modest 
probability and transform it into an algorithm that succeeds with overwhelming probability. In 
fact, even for moderate values of n an error probability that is of the order of 27” is so small that 
for all practical purposes, probabilistic algorithms are just as good as deterministic algorithms. 

If the original probabilistic algorithm used m coins, then the error reduction procedure we use 
(run k independent trials and output the majority answer) takes O(m - k) random coins to reduce 
the error to a value exponentially small in k. It is somewhat surprising that we can in fact do 
better, and reduce the error to the same level using only O(m + k) random bits (see Section 7.5). 
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NOTE 7.13 (THE CHERNOFF BOUND) 

The Chernoff bound is extensively used (sometimes under different names) 
in many areas of computer science and other sciences. A typical scenario is 
the following: there is a universe U of objects, a fraction u of them have a 
certain property, and we wish to estimate u. For example, in the proof of 
Theorem 7.10 the universe was the set of 2” possible coin tosses of some 
probabilistic algorithm and we wanted to know how many of them cause the 
algorithm to accept its input. Another example is that U may be the set of 
all the citizens of the United States, and we wish to find out how many of 
them approve of the current president. 

A natural approach to compute the fraction p is to sample n members of the 
universe independently at random, find out the number & of the sample’s 
members that have the property and to estimate that u is k/n. Of course, 
it may be quite possible that 10% of the population supports the president, 
but in a sample of 1000 we will find 101 and not 100 such people, and so 
we set our goal only to estimate y up to an error of +e for some e > 0. 
Similarly, even if only 10% of the population have a certain property, we 
may be extremely unlucky and select only people having it for our sample, 
and so we allow a small probability of failure 6 that our estimate will be 
off by more than e. The natural question is how many samples do we need 
to estimate u up to an error of +e with probability at least 1 — ô? The 
Chernoff bound tells us that (considering y as a constant) this number is 
O(log (1/5)/€?). 

This implies that if we sample n elements, then the probability that the 
number k having the property is py/n far from un decays exponentially with 
p: that is, this probability has the famous “bell curve” shape: 


k have 


Pri property 


un-pn 2 pn nt pnt? 


k 


We will use this exponential decay phenomena several times in this book, 
starting with the proof of Theorem 7.17, showing that BPP C P/poly. 
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7.4.2 Expected running time versus worst-case running time. 


When defining RTIME(T(n)) and BPTIME(T(n)) we required the machine to halt in T(n) time 
regardless of its random choices. We could have used expected running time instead, as in the 
definition of ZPP (Definition 7.7). It turns out this yields an equivalent definition: we can add a 
time counter to a PTM M whose expected running time is T(n) and ensure it always halts after 
at most 1007 (n) steps. By Markov’s inequality (see Lemma A.10), the probability that M runs 
for more than this time is at most 1/100. Thus by halting after 1007 (n) steps, the acceptance 
probability is changed by at most 1/100. 


7.4.3 Allowing more general random choices than a fair random coin. 


One could conceive of real-life computers that have a “coin” that comes up heads with probability 
p that is not 1/2. We call such a coin a p-coin. Indeed it is conceivable that for a random source 
based upon quantum mechanics, p is an irrational number, such as 1/e. Could such a coin give 
probabilistic algorithms new power? The following claim shows that it will not. 


LEMMA 7.14 
A coin with Pr[Heads| = p can be simulated by a PTM in expected time O(1) provided the ith bit 
of p is computable in poly(i) time. 


PROOF: Let the binary expansion of p be 0.pipap3.... The PTM generates a sequence of random 

bits b1,b9,..., one by one, where b; is generated at step 1. If b; < p; then the machine outputs 

“heads” and stops; if b; > p; the machine outputs “tails” and halts; otherwise the machine goes 

to step i + 1. Clearly, the machine reaches step ¿+ 1 iff bj = p; for all j < 2, which happens with 

probability 1/2’. Thus the probability of “heads” is em Piar, which is exactly p. Furthermore, the 
1 


expected running time is > 7,1% + 37. For every constant c this infinite sum is upperbounded by 


another constant (see Exercise 1). W 
Conversely, probabilistic algorithms that only have access to p-coins do not have less power 
than standard probabilistic algorithms: 


LEMMA 7.15 (VON-NEUMANN) 

A coin with Pr[Heads] = 1/2 can be simulated by a probabilistic TM with access to a stream of 
p-biased coins in expected time Ola) 

PROOF: We construct a TM M that given the ability to toss p-coins, outputs a 1/2-coin. The 
machine M tosses pairs of coins until the first time it gets two different results one after the other. 
If these two results were first “heads” and then “tails”, M outputs “heads”. If these two results 
were first “tails” and then “heads”, M outputs “tails”. For each pair, the probability we get two 
“heads” is p?, the probability we get two “tails” is (1 — p)?, the probability we get “head” and 
then “tails” is p(1 — p), and the probability we get “tails” and then “head” is (1 — p)p. We see that 
the probability we halt and output in each step is 2p(1 — p), and that conditioned on this, we do 
indeed output either “heads” or “tails” with the same probability. Note that we did not need to 
know p to run this simulation. W 
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Weak random sources. Physicists (and philosophers) are still not completely certain that ran- 
domness exists in the world, and even if it does, it is not clear that our computers have access to 
an endless stream of independent coins. Conceivably, it may be the case that we only have access 
to a source of imperfect randomness, that although unpredictable, does not consist of independent 
coins. As we will see in Chapter 16, we do know how to simulate probabilistic algorithms designed 
for perfect independent 1/2-coins even using such a weak random source. 


7.5 Randomness efficient error reduction. 


In Section 7.4.1 we saw how we can reduce error of probabilistic algorithms by running them 
several time using independent random bits each time. Ideally, one would like to be frugal with 
using randomness, because good quality random number generators tend to be slower than the rest 
of the computer. Surprisingly, the error reduction can be done just as effectively without using 
truly independent runs, and “recycling” the random bits. Now we outline this idea; a much more 
general theory will be later presented in Chapter 16. 

The main tool we use is expander graphs. Expander graphs have played a crucial role in nu- 
merous computer science applications, including routing networks, error correcting codes, hardness 
of approximation and the PCP theorem, derandomization, and more. Expanders can be defined 
in several roughly equivalent ways. One is that these are graphs where every set of vertices has a 
very large boundary. That is, for every subset S of vertices, the number of S’s neighbors outside 
S is (up to a constant factor) roughly equal to the number of vertices inside S. (Of course this 
condition cannot hold if S is too big and already contains almost all of the vertices in the graph.) 
For example, the n by n grid (where a vertex is a pair (i,j) and is connected to the four neighbors 
(¿+1,j+1)) is not an expander, as any k by k square (which is a set of size k?) in this graph only 
has a boundary of size O(k) (see Figure 7.1). 


Expander: no. of S's neighbors = Omega(|S|) Grid is not an expander: 
no. of S's neighbors = o(/s|"/2) 


Figure 7.1: In a combinatorial expander, every subset S of the vertices that is not too big has at least (|S!) 
neighbors outside the set. The grid (and every other planar graph) is not a combinatorial expander as a k x k square 
in the grid has only O(k) neighbors outside it. 


We will not precisely define expanders now (but see Section 7.B at the end of the chapter). 
However, an expander graph family is a sequence of graphs {Gy} wen such for every N, Gy is an 
N-vertex D-degree graph for some constant D. Deep mathematics (and more recently, simpler 
mathematics) has been used to construct expander graphs. These constructions yield algorithms 
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that, given the binary representation of N and an index of a node in Gy, can produce the indices 
of the D neighbors of this node in poly(log N) time. 

We illustrate the error reduction procedure by showing how we transform an RP algorithm that 
outputs the right answer with probability 1/2into an algorithm that outputs the right answer with 
probability 1 — 2-2), The idea is simple: let x be an input, and suppose we have an algorithm M 
using m coins such that if x € L then Pr,e (0139 [M(x,r) = 1] > 1/2 and if x ¢ L then M(x,r) =0 
for every r. Let N = 2” and let Gy be an N-vertex expander family. We use m coins to select a 
random vertex v from Gy, and then use log Dk coins to take a k — 1-step random walk from v on 
Gy. That is, at each step we choose a random number i in [D] and move from the current vertex 
to its it” neighbor. Let v,,...,v% be the vertices we encounter along this walk (where vı = v). We 
can treat these vertices as elements of {0,1} and run the machine M on input x with all of these 
coins. If even one of these runs outputs 1, then output 1. Otherwise, output 0. It can be shown 
that if less than half of the r’s cause M to output 0, then the probability that the walk is fully 
contained in these “bad” r’s is exponentially small in k. 

We see that what we need to prove is the following theorem: 


THEOREM 7.16 

Let G be an expander graph of N vertices and B a subset of G’s vertices of size at most GN, where 
B <1. Then, the probability that a k-vertex random walk is fully contained in B is at most p", 
where p < 1 is a constant depending only on 3 (and independent of k). 


Theorem 7.16 makes intuitive sense, as in an expander graph a constant fraction of the edges 
adjacent to vertices of B will have the other vertex in B’s complement, and so it seems that at each 
step we will have a constant probability to leave B. However, its precise formulation and analysis 
takes some care, and is done at the end of the chapter in Section 7.B. 

Intuitively, We postpone the full description of the error reduction procedure and its analysis 
to Section 7.B. 


7.6 BPP C P/poly 


Now we show that all BPP languages have polynomial sized circuits. Together with Theorem ?? 
this implies that if 3SAT € BPP then PH = *f. 


THEOREM 7.17 (ADLEMAN) 


PROOF: Sup- 
BPP C P/poly. 


pose L € BPP, then by the alternative definition of BPP and the error reduction procedure of 
Theorem 7.10, there exists a TM M that on inputs of size n uses m random bits and satisfies 


a € L => Pr,[M(z,r) accepts ] > 1 — 27 (+?) 
a g L => Pr, [M(a,r) accepts ] < 27+?) 
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(Such a machine exists by the error reduction arguments mentioned earlier.) 

Say that an r € {0,1}” is bad for an input x € {0,1}” if M(x,r) is an incorrect answer, 
otherwise we say its good for x. For every x, at most 2-2”"/ 2("+2) values of r are bad for x. Adding 
over all x € (0,1)”, we conclude that at most 2” x 2/2("+)) = 2™/2 strings r are bad for some 
x. In other words, at least 2™ — 2/2 choices of r are good for every x € {0,1}". Given a string 
ro that is good for every x € {0,1}", we can hardwire it to obtain a circuit C (of size at most 
quadratic in the running time of M) that on input x outputs M(z,ro). The circuit C will satisfy 
C(x) = L(x) for every x € [0,1)”. E 


7.7 BPP is in PH 


At first glance, BPP seems to have nothing to do with the polynomial hierarchy, so the next 
theorem is somewhat surprising. 


THEOREM 7.18 (SIPSER-GACS) 
BPP C En 1 


PROOF: It is enough to prove that BPP C Xf because BPP is closed under complementation (i.e., 
BPP = coBPP). 

Suppose L € BPP. Then by the alternative definition of BPP and the error reduction proce- 
dure of Theorem 7.10 there exists a polynomial-time deterministic TM M for L that on inputs of 
length n uses m = poly(n) random bits and satisfies 


x € L => Pr, |M(z,r) accepts | > 1-2” 
x g L => Pr, |M(z2,r) accepts ] < 27” 
For x € {0,1}”, let Sẹ denote the set of r’s for which M accepts the input pair (x,r). Then 


either |S,| > (1—27")2™ or |Sz| < 27727, depending on whether or not x € L. We will show how 
to check, using two alternations, which of the two cases is true. 


LD 


Figure 7.2: There are only two possible sizes for the set of r’s such that M(x,r) =Accept: either this set is almost 
all of £0,1)”” or a tiny fraction of (0,1)'””. In the former case, a few random “shifts” of this set are quite likely to 
cover all of (0, 1)”. In the latter case the set’s size is so small that a few shifts cannot cover (0, 1)” 


For k = % 4 1, let U = (uz, ...,up) be a set of k strings in (0,1). We define Gy to be a 
graph with vertex set [0,1)'” and edges (r,s) for every r,s such that r = s + u; for some i € [k] 


p7.14 (128) 7.8. STATE OF OUR KNOWLEDGE ABOUT BPP 


(where + denotes vector addition modulo 2, or equivalently, bitwise XOR). Note that the degree 
of Gy is k. For a set S C ([0,1)””, define [y(S) to be all the neighbors of S in the graph Gy. That 
is, r € Ty(S) if there is an s € S and i € [k] such that r= s+ uj. 
Claim 1: For every set S C {0,1} with |S| < 2™-” and every set U of size k, it holds that 
Ty(S) 4 {0,1}. Indeed, since Ty has degree k, it holds that |[y($)| < k|S| < 2”. 
Claim 2: For every set S C {0,1} with |S| > (1 — 27”)2™ there exists a set U of size k such that 
Ty(S) = {0,1}"". We show this by the probabilistic method, by proving that for every S, if we 
choose U at random by taking k random strings w1,..., ug, then Pr[Py(S) = {0,1}"] > 0. Indeed, 
for r € {0,1}, let B, denote the “bad event” that r is not in Py(S). Then, B, = Mex] Bi where 
B} is the event that r ¢ S + u;, or equivalently, that r + u; ¢ S (using the fact that modulo 2, 
at+b=c@a=c+b). Yet, r+u; is a uniform element in {0,1}’", and so it will be in S with 
probability at least 1 — 27”. Since B!,...,B* are independent, the probability that B, happens is 
at most (1 — 27”)¥ < 2, By the union bound, the probability that Ty(S) 4 (0, 1F” is bounded 
by Dore {0,1}™ Pr[B,] < 1. 

Together Claims 1 and 2 show x € L if and only if the following statement is true 


k 
duy,...,uz € {0,1} Vr e {0,1}” V M(z,r O u;)accepts 


i=1 


thus showing L € X2. E 


7.8 State of our knowledge about BPP 


We know that P C BPP C P/poly, and furthermore, that BPP C X5N IB and so if NP = p 
then BPP = P. As mentioned above, there are complexity-theoretic reasons to strongly believe 
that BPP C DTIME(2‘) for every e > 0, and in fact to reasonably suspect that BPP = P (see 
Chapters 16 and 17). However, currently we are not even able to rule out that BPP = NEXP! 


Complete problems for BPP? 


Though a very natural class, BPP behaves differently in some ways from other classes we have 
seen. For example, we know of no complete languages for it (under deterministic polynomial time 
reductions). One reason for this difficulty is that the defining property of BPTIME machines is 
semantic, namely, that for every string they either accept with probability at least 2/3 or reject 
with probability at least 1/3. Given the description of a Turing machine M, testing whether it has 
this property is undecidable. By contrast, the defining property of an NDTM is syntactic: given a 
string it is easy to determine if it is a valid encoding of an NDTM. Completeness seems easier to 
define for syntactically defined classes than for semantically defined ones. For example, consider 
the following natural attempt at a BPP-complete language: L = { (M, x) : Pr[M(x) = 1] > 2/3). 
This language is indeed BPP-hard but is not known to be in BPP. In fact, it is not in any level 
of the polynomial hierarchy unless the hierarchy collapses. We note that if, as believed, BPP = P, 
then BPP does have a complete problem. (One can sidestep some of the above issues by using 
promise problems instead of languages, but we will not explore this.) 
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Does BPTIME have a hierarchy theorem? 


Is BPTIME(n*) contained in BPTIME(n) for some c > 1? One would imagine not, and this 
seems as the kind of result we should be able to prove using the tools of Chapter 3. However 
currently we are even unable to show that BPTIME(nbe””) (say) is not in BPTIME(n). The 
standard diagonalization techniques fail, for similar reasons as the ones above. However, recently 
there has been some progress on obtaining hierarchy theorem for some closely related classes (see 
notes). 


7.9 Randomized reductions 


Since we have defined randomized algorithms, it also makes sense to define a notion of random- 
ized reduction between two languages. This proves useful in some complexity settings (e.g., see 
Chapters 9 and 8). 


DEFINITION 7.19 
Language A reduces to language B under a randomized polynomial time reduction, denoted A <, B, 
if there is a probabilistic TM M such that for every x € {0,1}*, Pr[B(M(x)) = A(x)] > 2/s. 


We note that if A <, B and B € BPP then A € BPP. This alerts us to the possibility that we 
could have defined NP-completeness using randomized reductions instead of deterministic reduc- 
tions, since arguably BPP is as good as P as a formalization of the notion of efficient computation. 
Recall that the Cook-Levin theorem shows that NP may be defined as the set {L : L <p 3SAT}. 
The following definition is analogous. 


DEFINITION 7.20 (BP - NP) 
BP -NP = {L : L <, 3SAT). 


We explore the properties of BP-NP in the exercises, including whether or not 35AT € BP-NP. 

One interesting application of randomized reductions will be shown in Chapter 9, where we 
present a (variant of a) randomized reduction from 3SAT to the solving special case of 3SAT 
where we are guaranteed that the formula is either unsatisfiable or has a single unique satisfying 
assignment. 


7.10 Randomized space-bounded computation 


A PTM is said to work in space S(n) if every branch requires space O(S(n)) on inputs of size n and 
terminates in 22(5(") time. Recall that the machine has a read-only input tape, and the work space 
only cell refers only to its read/write work tapes. As a PTM it has two transition functions that 
are applied with equal probability. The most interesting case is when the work tape has O(log n) 
size. The classes BPL and RL are the two-sided error and one-sided error probabilistic analogs of 
the class L defined in Chapter 4. 
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DEFINITION 7.21 ([) 

The classes BPL and RL] A language L is in BPL if there is an O(log n)-space 
probabilistic TM M such that Pr[M(x) = L(x)] > 2/3. 

A language L is in RL if there is an O(log n)-space probabilistic TM M such that 
if x € L then Pr[M(x) = 1] > 2/3 and if x ¢ L then Pr[M(x) = 1] = 0. 


The reader can verify that the error reduction procedure described in Chapter 7 can be imple- 
mented with only logarithmic space overhead, and hence also in these definitions the choice of the 
precise constant is not significant. We note that RL C NL, and thus RL C P. The exercises ask 
you to show that BPL C P as well. 


One famous RL-algorithm is the algorithm to solve UPATH: the restriction of the NL-complete 
PATH problem (see Chapter 4) to undirected graphs. That is, given an n-vertex undirected graph 
G and two vertices s and t, determine whether s is connected to t in G. 


THEOREM 7.22 ([AKL*79]) 
UPATH € RL. 


The algorithm for UPATH is actually very simple: take a random walk of length n? starting 
from s. That is, initialize the variable v to the vertex s and in each step choose a random neighbor 
u of v, and set v — u. Accept iff the walk reaches t within n? steps. Clearly, if s is not connected to 
t then the algorithm will never accept. It can be shown that if s is connected to t then the expected 
number of steps it takes for a walk from s to hit t is at most an and hence our algorithm will accept 
with probability at least 3, We defer the analysis of this algorithm to the end of the chapter at 
Section 7.A, where we will prove that a somewhat larger walk suffices to hit t with good probability 


(see also Exercise 9). 


In Chapter 16 we show a recent deterministic logspace algorithm for the same problem. It is 
known that BPL (and hence also RL) is contained in SPACE(log?/? n). In Chapter 16 we will 
see a somewhat weaker result: a simulation of BPL in log? n space and polynomial time. 
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WHAT HAVE WE LEARNED? 


e The class BPP consists of languages that can be solved by a probabilistic 
polynomial-time algorithm. The probability is only over the algorithm’s coins 
and not the choice of input. It is arguably a better formalization of efficient 
computation than P. 


RP,coRP and ZPP are subclasses of BPP corresponding to probabilistic 
algorithms with one-sided and “zero-sided” error. 


Using repetition, we can considerably amplify the success probability of prob- 
abilistic algorithms. 


e We only know that P C BPP C EXP, but we suspect that BPP = P. 


e BPP is a subset of both P/poly and PH. In particular, the latter implies 
that if NP = P then BPP =P. 


e Randomness is used in complexity theory in many contexts beyond BPP. Two 
examples are randomized reductions and randomized logspace algorithms, but 
we will see many more later. 


Chapter notes and history 


Early researchers realized the power of randomization since their computations —e.g., for design 
of nuclear weapons— used probabilistic tools such as Monte Carlo simulations. Papers by von 
Neumann [von61] and de Leeuw et al. [LMSS56] describe probabilistic Turing machines. The 
definitions of BPP, RP and ZPP are from Gill [Gil77]. (In an earlier conference paper [Gil74], 
Gill studies similar issues but seems to miss the point that a practical algorithm for deciding a 
language must feature a gap between the acceptance probability in the two cases.) 

The algorithm used to show PRIMES is in coRP is due to Solovay and Strassen [SS77]. Another 
primality test from the same era is due to Rabin [Rab80]. Over the years, better tests were proposed. 
In a recent breakthrough, Agrawal, Kayal and Saxena finally proved that PRIMES € P. Both the 
probabilistic and deterministic primality testing algorithms are described in Shoup’s book [?]. 

Lovász's randomized NC algorithm [Lov79] for deciding the existence of perfect matchings is 
unsatisfying in the sense that when it outputs “Accept,” it gives no clue how to find a matching! 
Subsequent probabilistic NC algorithms can find a perfect matching as well; see [KUW86, MVV87]. 

BPP C P/poly is from Adelman [Ad178]. BPP C PH is due to Sipser [Sip83], and the stronger 
form BPP C IT is due to P. Gács. Recent work [] shows that BPP is contained in classes 
that are seemingly weaker than X5 N IT}. 

Even though a hierarchy theorem for BPP seems beyond our reach, there has been some success 
in showing hierarchy theorems for the seemingly related class BPP/1 (i.e., BPP with a single bit 
of nonuniform advice) [Bar02, ?, ?]. 
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Readers interested in randomized algorithms are referred to the books by Mitzenmacher and 
Upfal [MU05] and Motwani and Raghavan [MR95]. 

STILL A LOT MISSING 

Expanders were well-studied for a variety of reasons in the 1970s but their application to 
pseudorandomness was first described by Ajtai, Komlos, and Szemeredi [AKS87]. Then Cohen- 
Wigderson [CW89] and Impagliazzo-Zuckerman (1989) showed how to use them to “recycle” ran- 
dom bits as described in Section 7.B.3. The upcoming book by Hoory, Linial and Wigderson (draft 
available from their web pages) provides an excellent introduction to expander graphs and their 
applications. 

The explicit construction of expanders is due to Reingold, Vadhan and Wigderson [RVWO00], 
although we chose to present it using the replacement product as opposed to the closely related 
zig-zag product used there. The deterministic logspace algorithm for undirected connectivity is due 
to Reingold [?]. 


Exercises 


$1 Show that for every c > 0, the following infinite sum is finite: 


qo 


oí 
i>l 


§2 Show, given input the numbers a,n, p (in binary representation), how to compute a”(modp) 
in polynomial time. 
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83 Let us study to what extent Claim ?? truly needs the assumption that p is efficiently com- 
putable. Describe a real number p such that given a random coin that comes up “Heads” 
with probability p, a Turing machine can decide an undecidable language in polynomial time. 
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84 Show that ZPP = RP NcoRP. 


$5 A nondeterministic circuit has two inputs x,y. We say that it accepts x iff there exists y 
such that C(x, y) = 1. The size of the circuit is measured as a function of |x|. Let NP /poly 
be the languages that are decided by polynomial size nondeterministic circuits. Show that 
BP-NP C NP /poly. 


$6 Show using ideas similar to the Karp-Lipton theorem that if 3SAT € BP - NP then PH 
collapses to X$. (Combined with above, this shows it is unlikely that 3SAT <, 3SAT.) 


87 Show that BPL C P 
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$8 Show that the random walk idea for solving connectivity does not work for directed graphs. 
In other words, describe a directed graph on n vertices and a starting point s such that the 
expected time to reach t is (2(2”) even though there is a directed path from s to t. 


$9 Let G be an n vertex graph where all vertices have the same degree. 


(a) 


We say that a distribution p over the vertices of G (where p; denotes the probability 
that vertex i is picked by p) is stable if when we choose a vertex i according to p and 
take a random step from i (i.e., move to a random neighbor j or i) then the resulting 
distribution is p. Prove that the uniform distribution on G’s vertices is stable. 


For p be a distribution over the vertices of G, let A(p) = max;{p; — 1/n}. For every 
k, denote by p* the distribution obtained by choosing a vertex i at random from p and 
taking k random steps on G. Prove that if G is connected then there exists k such that 
A(p*) < (1 —n7!0")A(p). Conclude that 

i. The uniform distribution is the only stable distribution for G. 

ii. For every vertices u,v of G, if we take a sufficiently long random walk starting from 
u, then with high probability the fraction of times we hit the vertex v is roughly 
1/n. That is, for every e > 0, there exists k such that the k-step random walk from 
u hits v between (1 — e)k/n and (1 +e)k/n times with probability at least 1 — e. 


For a vertex u in G, denote by E,, the expected number of steps it takes for a random 
walk starting from u to reach back u. Show that E,, < 10n?. 
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For every two vertices u,v denote by Eu w the expected number of steps it takes for 
a random walk starting from u to reach v. Show that if u and v are connected by a 
path of length at most k then Ey. < 100kn?. Conclude that for every s and t that are 
connected in a graph G, the probability that an 1000n? random walk from s does not 
hit t is at most 1/10. 
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Let G be an n-vertex graph that is not necessarily regular (i.e., each vertex may have 
different degree). Let G’ be the graph obtained by adding a sufficient number of parallel 
self-loops to each vertex to make G regular. Prove that if a k-step random walk in G’ 
from a vertex s hits a vertex t with probability at least 0.9, then a 10n?k-step random 
walk from s will hit t with probability at least 1/2. 
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The following exercises are based on Sections 7.A and 7.B. 


$10 Let A be a symmetric stochastic matrix: A = A! and every row and column of A has non- 
negative entries summing up to one. Prove that ||Al| < 1. 
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$11 Let A, B be two symmetric stochastic matrices. Prove that \(A + B) < (A) + A(B). 


812 Let a n, d random graph be an n-vertex graph chosen as follows: choose d random permuta- 
tions 7, ldots,7q from [n] to [n]. Let the the graph G contains an edge (u,v) for every pair 
u,v such that v = 7;(u) for some 1 < i < d. Prove that a random n,d graph is an (n, 2d, 3d) 
combinatorial expander with probability 1 — o(1) (i.e., tending to one with n). 


“3 KIOAO 
103 1,5 (S)'u yeyy Ayiqeqosd oy} punog oy £14 ‘|¢|(pE + 1) > |l 
ym [u] 5 7 ys pue q/u > |g] ym u 5 y Jos £249 10] :JUTH 


7.A. RANDOM WALKS AND EIGENVALUES p7.21 (135) 


The following two section assume some knowledge of elementary linear algebra (vector spaces and 
Hilbert spaces); see Appendix A for a quick review. 


7.A Random walks and eigenvalues 


In this section we study random walks on (undirected regular) graphs, introducing several important 
notions such as the spectral gap of a graph’s adjacency matrix. As a corollary we obtain the proof 
of correctness for the random-walk space-efficient algorithm for UPATH of Theorem 7.22. We will 
see that we can use elementary linear algebra to relate parameters of the graph’s adjacency matrix 
to the behavior of the random walk on that graph. 


REMARK 7.23 
In this section, we restrict ourselves to regular graphs, in which every vertex have the same degree, 
although the definitions and results can be suitably generalized to general (non-regular) graphs. 


7.A.1 Distributions as vectors and the parameter A(G). 


Let G be a d-regular n-vertex graph. Let p be some probability distribution over the vertices of G. 
We can think of p as a (column) vector in R” where p; is the probability that vertex 7 is obtained 
by the distribution. Note that the Lj-norm of p (see Note 7.24), defined as |p|, = X; |pil, is 
equal to 1. (In this case the absolute value is redundant since p; is always between 0 and 1.) 

Now let q represent the distribution of the following random variable: choose a vertex i in G 
according to p, then take a random neighbor of 7 in G. We can compute q as a function of p: the 
probability q; that j is chosen is equal to the sum over all 7’s neighbors 7 of the probability p; that 
i is chosen times 1/d (where 1/d is the probability that, conditioned on i being chosen, the walk 
moves to q). Thus q = Ap, where A = A(G) which is the normalized adjacency matriz of G. That 
is, for every two vertices 7,7, A; j is equal to the number of edges between 7 and j divided by d. 
Note that A is a symmetric matrix,? where each entry is between 0 and 1, and the sum of entries 
in each row and column is exactly one (such a matrix is called a symmetric stochastic matrix). 

Let {e’}"™, be the standard basis of R” (i.e. e? has 1 in the it? coordinate and zero everywhere 
else). Then, ATe® represents the distribution Xr of taking a T-step random walk from the vertex 
s. This already suggests that considering the adjacency matrix of a graph G could be very useful 
in analyzing random walks on G. 


DEFINITION 7.25 (THE PARAMETER A(G).) 

Denote by 1 the vector (1/n,1/n,...,1/n) corresponding to the uniform distri- 
bution. Denote by 1+ the set of vectors perpendicular to 1 (i.e, v € 1+ if 
(v,1) = (1/n) E, vi=0). 

The parameter (A), denoted also as A(G), is the maximum value of || Av||, over all 
vectors v € 1+ with ||v||, = 1. 


3A matrix A is symmetric if A= A‘, where A! denotes the transpose of A. That is, (A), ; = Aj, for every i, j. 
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NOTE 7.24 (Lp NORMS) 

A norm is a function mapping a vector v into a real number ||v|| satisfying 
(1) ||v|| > 0 with ||v|| = 0 if and only v is the all zero vector, (2) |lav|| = 
|a| - ||v|| for every a € R, and (3) [lv + ul] < ||v|| + [ul] for every vector u. 
The third inequality implies that for every norm, if we define the distance 
between two vectors u,v as ||u— v|| then this notion of distance satisfies the 
triangle inequality. 


For every v € R” and number p > 1, the Lp norm of v, denoted ||v||,,, is equal 
to (Xa [vp P One particularly interesting case is p = 2, the so-called 


Euclidean norm, in which ||v||, =1/M;_, v2 = y (v, v). Another interesting 


case is p = 1, where we use the single bar notation and denote |v|, = 
J; |vi|. Another case is p = 00, where we denote ||v||,, = limp. ||v||, 


Maxje[n] (Vil. 


The Hélder inequality says that for every p,q with 7 + i = 1, Jlull,llvil, = 
X; lu;v;]. To prove it, note that by simple scaling, it suffices to con- 
sider norm one vectors, and so it enough to show that if [Jul], = [lvl], = 1 


then Dz, |uillvi| < 1. But y fuillvi] = Da upv 0/0 < 
a 3 Ju]? + alvil? E i + i = 1, where the last inequality uses the fact 
that for every a,b > 0 and a € [0,1], a®b'~* < aa + (1—a)b. This fact is 
due to the log function being concave— having negative second derivative, 
implying that aloga + (1 — a) logb < log(aa + (1 — a)b). 

Setting p = 1 and q = œ, the Holder inequality implies that 

lvl < lvl, lvl 

Setting p = q = 2, the Hölder inequality becomes the Cauchy- 
Schwartz Inequality stating that X; Jusvi] < |lull,||v||,. Setting u = 


(1//n,1/yn,...,1/yn), we get that 


n 
[v],/Vn => lvl < lvli 
i=1 
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REMARK 7.26 

The value A(G) is often called the second largest eigenvalue of G. The reason is that since A is a 
symmetric matrix, we can find an orthogonal basis of eigenvectors v!,...,v” with corresponding 
eigenvalues A1,..., An which we can sort to ensure |\i| > [A2]... > [An]. Note that 41 = 1. 
Indeed, for every i, (41); is equal to the inner product of the it row of A and the vector 1 which 
(since the sum of entries in the row is one) is equal to 1/n. Thus, 1 is an eigenvector of A with 
the corresponding eigenvalue equal to 1. One can show that a symmetric stochastic matrix has 
all eigenvalues with absolute value at most 1 (see Exercise 10) and hence we can assume A; = 1 
and vt = 1. Also, because 1+ = Span{v?,...,v”}, the value A above will be maximized by (the 
normalized version of) v?, and hence A(G) = |A2|. The quantity 1 — A(G) is called the spectral gap 
of the graph. We note that some texts use un-normalized adjacency matrices, in which case A(G) 
is a number between 0 and d and the spectral gap is defined to be d— A(G). 


One reason that \(G) is an important parameter is the following lemma: 


LEMMA 7.27 
For every regular n vertex graph G = (V, E) let p be any probability distribution over V, then 


144p = tlla < à” 


PROOF: By the definition of A(G), || Av||, < Allv||, for every v L 1. Note that if v L 1 then 
Av L 1 since (1, Av) = (AT1,v) = (1,v) = 0 (as A = At and A1 = 1). Thus A maps the space 
1+ to itself and since it shrinks any member of this space by at least A, A(AT) < A(A)T. (In fact, 
using the eigenvalue definition of A, it can be shown that A(47) = \(A).) 

Let p be some vector. We can break p into its components in the spaces parallel and orthogonal 
to 1 and express it as p = al +p’ where p’ L 1 and a is some number. If p is a probability 
distribution then a = 1 since the sum of coordinates in p’ is zero. Therefore, 


A’p= AT(1 +p)=1+ Arp! 


Since 1 and p’ are orthogonal, ||p||? = ||1||? + ||p’||? and in particular ||p’||, < ||p||,. Since p is 
a probability vector, ||p||, < |p|, -1< 1 (see Note 7.24). Hence ||p’||, < 1 and 


14%p — 1p = ATP ll, < A” 


It turns out that every connected graph has a noticeable spectral gap: 


LEMMA 7.28 
For every d-regular connected G with self-loops at each vertex, MG) < 1 — zzz. 


PROOF: Let u L 1 be a unit vector and let v = Au. We’ll show that 1 —||v||? > TE which implies 


Ivl? < 1 — ¿25 and hence ||v||, < 1— a: 
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Since ||u||, = 1, 1— ||v||? = llull? — ||v||2. We claim that this is equal to ij Aig Ui — vj)? 
where 7,7 range from 1 to n. Indeed, 


Y Aig (ui — vg)? = DO Ai gu? — 250 Aa guivs +) Aivi = 
ij ij ij ij 
lull? — 2(Au, v) + Ivl} = llull? 21v} + Ivl 


where these equalities are due to the sum of each row and column in A equalling one, and because 
Ivl? = (v, v) = (Au, v) = X; ¿ As juvy. 

Thus it suffices to show >, ; A; j(U; — vj)? > aks. This is a sum of non-negative terms so it 
suffices to show that for some i,j, As ¿(u; — vj)? > T First, because we have all the self-loops, 
Asi > 1/d for all i, and so we can assume |u; — vi| < ss for every i € [n], as otherwise we'd be 
done. 

Now sort the coordinates of u from the largest to the smallest, ensuring that uy > ug > -:: Un. 
Since }>, u; = 0 it must hold that u; > 0 > u,. In fact, since u is a unit vector, either uy > 1/yn 
or Un < 1/./n and so uj — un > 1/yn. One of the n— 1 differences between consecutive coordinates 
u; — Uj+1 must be at least 1/n!° and so there must be an ig such that if we let S = {1,..., io} 
and S = [n] \ S;, then for every i € S and j € S, u; — u; > 1/n!?. Since G is connected there 
exists an edge (i,j) between S and S. Since |v; — uj| < xs, for this choice of i,j, |u; — v| > 
hu; uj| 1 > 1 . Thus A; j (ui — vj}? > 1t E 


2n1.5 — 2nl5 = d4n3° 


REMARK 7.29 

The proof can be strengthened to show a similar result for every connected non-bipartite graph 
(not just those with self-loops at every vertex). Note that this condition is essential: if A is the 
adjacency matrix of a bipartite graph then one can find a vector v such that Av = —v. 


7.A.2 Analysis of the randomized algorithm for undirected connectivity. 


Together, Lemmas 7.27 and 7.28 imply that, at least for regular graphs, if s is connected to t then 
a sufficiently long random walk from s will hit t in polynomial time with high probability. 


COROLLARY 7.30 

Let G be a d-regular n-vertex graph with all vertices having a self-loop. Let s be a vertex in G. 
Let T > 10dn? logn and let Xr denote the distribution of the vertex of the T** step in a random 
walk from s. Then, for every j connected to s, Pr|Xr = j] > E. 


PROOF: By these Lemmas, if we consider the restriction of an n-vertex graph G to the connected 
component of s, then for every probability vector p over this component and T > 10dn? log n, 
||A7p — 1], < = (where 1 here is the uniform distribution over this component). Using the 
relations between the Lı and Lo norms (see Note 7.24), |ATp — 1|, < + and hence every element 
in the connected component appears in ATp with at least 1/n —1/(2n) > 1/(2n) probability. W 


Note that Corollary 7.30 implies that if we repeat the 10dn? log n walk for 10n times (or equiv- 
alently, if we take a walk of length 100dn* logn) then we will hit t with probability at least 3/4. 
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7.B Expander graphs. 


Expander graphs are extremely useful combinatorial objects, which we will encounter several times 
in the book. They can be defined in two equivalent ways. At a high level, these two equivalent 
definitions can be described as follows: 


e Combinatorial definition: A constant-degree regular graph G is an expander if for every subset 
S of less than half of G’s vertices, a constant fraction of the edges touching S are from S to 
its complement in G. This is the definition alluded to in Section 7.5 (see Figure 7.1).4 


e Algebraic expansion: A constant-degree regular graph G is an expander if its parameter \(G) 
bounded away from 1 by some constant. That is, A(G) < 1 — e for some constant e > 03 


What do we mean by a constant? By constant we refer to a number that is independent of the size 
of the graph. We will typically talk about graphs that are part of an infinite family of graphs, and 
so by constant we mean a value that is the same for all graphs in the family, regardless of their 
size. 

Below we make the definitions more precise, and show their equivalence. We will then complete 
the analysis of the randomness efficient error reduction procedure described in Section 7.5. 


7.B.1 The Algebraic Definition 


The Algebraic definition of expanders is as follows: 


DEFINITION 7.31 ((n, d, A)-GRAPHS.) 

If G is an n-vertex d-regular G with A(G) < A for some number A < 1 then we say 
that G is an (n, d, A)-graph. 

A family of graphs {Gn nen is an expander graph family if there are some constants 
d € N and à < 1 such that for every n, Gn is an (n, d, A)-graph. 


Explicit constructions. We say that an expander family {G,}nen is explicit if there is a 
polynomial-time algorithm that on input 1” with n € J outputs the adjacency matrix of Gp. 
We say that the family is strongly explicit if there is a polynomial-time algorithm that for every 
n € I on inputs (n,v,i) where 1 < v < n’ and 1 <i < d outputs the 7” neighbor of v. (Note that 
the algorithm runs in time polynomial in the its input length which is polylogarithmic in n.) 

As we will see below it is not hard to show that expander families exist using the probabilistic 
method. But this does not yield explicit (or very explicit) constructions of such graphs (which, as 
we saw in Section 7.4.1 are often needed for applications). In fact, there are also several explicit and 


“The careful reader might note that there we said that a graph is an expander if a constant fraction of S”s 
neighboring vertices are outside S. However, for constant-degree graphs these two notions are equivalent. 
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NOTE 7.33 (EXPLICIT CONSTRUCTION OF PSEUDORANDOM OBJECTS) 
Expanders are one instance of a recurring theme in complexity theory (and 
other areas of math and computer science): it is often the case that a ran- 
dom object can be easily proven to satisfy some nice property, but the ap- 
plications require an explicit object satisfying this property. In our case, 
a random d-regular graph is an expander, but to use it for, say, reducing 
the error of probabilistic algorithms, we need an explicit construction of an 
expander family, with an efficient deterministic algorithm to compute the 
neighborhood relations. Such explicit constructions can be sometimes hard 
to come by, but are often surprisingly useful. For example, in our case the 
explicit construction of expander graphs turns out to yield a deterministic 
logspace algorithm for undirected connectivity. 

We will see another instance of this theme in Chapter 17, which discusses 
error correcting codes. 


strongly explicit constructions of expander graphs known. The smallest can be for a d-regular 


n-vertex graph is Ua) and there are constructions meeting this bound (specifically the bound 


is (1 — TE where by o(1) we mean a function that tends to 0 as the number of vertices 
grows; graphs meeting this bound are called Ramanujan graphs). However, for most applications in 
Computer Science, any family with constant d and A < 1 will suffice (see also Remark 7.32 below). 
Some of these constructions are very simple and efficient, but their analysis is highly non-trivial 
and uses relatively deep mathematics.” In Chapter 16 we will see a strongly explicit construction 
of expanders with elementary analysis. This construction also introduces a tool that is useful to 
derandomize the random-walk algorithm for UPATH. 


REMARK 7.32 

One reason that the particular constants of an expander family are not extremely crucial is that we 
can improve the constant A (make it arbitrarily smaller) at the expense of increasing the degree: 
this follows from the fact, observed above in the proof of Lemma 7.27, that \(G7) = \(G)", where 
GT denotes the graph obtained by taking the adjacency matrix to the Tt? power, or equivalently, 
having an edge for every length-T path in Œ. Thus, we can transform an (n,d, A) graph into an 
(n, dT, AT)-graph for every T > 1. In Chapter 16 we will see a different transformation called 
the replacement product to decrease the degree at the expense of increasing A somewhat (and also 
increasing the number of vertices). 


5An example for such an expander is the following 3-regular graph: the vertices are the numbers 1 to p — 1 for 
some prime p, and each number x is connected to x + 1,2 — 1 and z™! (mod p). 
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7.B.2 Combinatorial expansion and existence of expanders. 


We describe now a combinatorial criteria that is roughly equivalent to Definition 7.31. One ad- 
vantage of this criteria is that it makes it easy to prove that a non-explicit expander family exists 
using the probabilistic method. It is also quite useful in several applications. 


DEFINITION 7.34 ([) 

Combinatorial (edge) expansion] An n-vertex d-regular graph G = (V, E) is called 
an (n,d,p)-combinatorial expander if for every subset S C V with |S| < n/2, 
|E(S,S)| > pd|S|, where for subsets S,T of V, E(S,T) denotes the set of edges 
(s,t) with s € S and tE T. 


Note that in this case the bigger pis the better the expander. We’ll loosely use the term expander 
for any (n, d, p)-combinatorial expander with c a positive constant. Using the probabilistic method, 
one can prove the following theorem: (Exercise 12 asks you to prove a slightly weaker version) 
THEOREM 7.35 (EXISTENCE OF EXPANDERS) 

Let e > 0 be some constant. Then there exists d = d(e) and N € N such that for every n > N 
there exists an (n, d, 1 — e)-combinatorial expander. 


The following theorem related combinatorial expansion with our previous Definition 7.31 


THEOREM 7.36 (COMBINATORIAL AND ALGEBRAIC EXPANSION) 
1. IfG is an (n, d, A)-graph then it is an (n, d, (1 — A)/2)-combinatorial expander. 


2. If G is an (n, d, p)-combinatorial expander then it is an (n,d,1 — 2 )-graph. 


The first part of Theorem 7.36 follows by plugging T = S into the following lemma: 


LEMMA 7.37 (EXPANDER MIXING LEMMA) 
Let G = (V, E) be an (n, d, A)-graph. Let S,T C V, then 


heso- Sisin] < avis 


PROOF: Let s denote the vector such that s; is equal to 1 if i € S and equal to 0 otherwise, and let 
t denote the corresponding vector for the set S. Thinking of s as a row vector and of t as a column 
vector, the Lemma’s statement is equivalent to 


[s At = An < AVĪSĪITĪ, (2) 


where A is G’s normalized adjacency matrix. Yet by Lemma 7.40, we can write A as (1—A)J+AC, 
where J is the matrix with all entries equal to 1/n and C has norm at most one. Hence, 


sAt =(1—A)sJt + Asct < MIEL + A y/[SIIT1, 
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where the last inequality follows from sJt = |S||T|/n and sCt = (s, Ct) < ||s||,||t||, = W|S||7|. E 


PROOF OF SECOND PART OF THEOREM 7.36.: We prove a slightly relaxed version, replacing 
the constant 2 with 8. Let G = (V, E) be an n-vertex d-regular graph such that for every subset 
S CV with |S| < n/2, there are p|S| edges between S and S = V \ S, and let A be G’s normalized 
adjacency matrix. 

Let A = A(G). We need to prove that A < 1 — p?/8. Using the fact that A is the second 
eigenvalue of A, there exists a vector u L 1 such that Au = Au. Write u = v + w where v is equal 
to u on the coordinates on which u is positive and equal to O otherwise, and w is equal to u on the 
coordinates on which u is negative, and equal to 0 otherwise. Note that, since u 1 1, both v and 
w are nonzero. We can assume that u is nonzero on at most n/2 of its coordinates (as otherwise 
we can take —u instead of u). 

Since Au = Au and (v,w) = 0, 


(Av, v) + (Aw,v) = (A(v + w), v) = (Au, v) = (A(v + w), v) = Allv|l2- 


Since (Aw, v) is negative, we get that (Av, v)/||v||? > A or 


(Av, v) _ lvl- (Av, v) _ Xij Aaa (vi — va)? 
lvli IMIA 2lIv Il 


1-1>1 


where the last equality is due to ig AV va)? z Dij Ai jv? —2 Dij Aij Vivi + Dij As ¿v5 = 
2\|v||? — 2(Av, v). (We use here the fact that each row and column of A sums to one.) Multiply 
both numerator and denominator by >, 5 Aig (vé + v5). By the Cauchy-Schwartz inequality,° we 
can bound the new numerator as follows: 

2 


Y Aas (vi — vs)? | |) Agvet vs)? | < | DD 4is(vi— va) (ve + v) 
ij ij ij 
Hence, using (a — b) (a + b) = a? — b?, 


2 2 
(Ziy Aij? — v2) (Ziy 4i5(v? — v3) 
Le TE A UN = 
Ilha aay Ar5(vi + v3) 2\|vll2 7 Aj ¿vi +2 ij A vivi + Doi; Azv?) 
2 2 
eae Aj (vi - v3)) Sa As ¡(vi - v3)) 
2\|v||2 (2lv2 + 2(Av, v)) — 8l|v 11: 


where the last inequality is due to A having matrix norm at most 1, implying (Av, v) < ||v||?. We 
conclude the proof by showing that 


Y Aig(v? — vj) 2 allvll, (3) 
i,j 


The Cauchy-Schwartz inequality is typically stated as saying that for x,y € R”, YN xy: < VO ¿XDO_, y). 
However, it is easily generalized to show that for every non-negative ¿41,..., Hn, >>; MiXiyi < YO), hix?) Q; uy?) 
(this can be proven from the standard Cauchy-Schwartz by multiplying each coordinate of x and y by \/pi. It is this 
variant that we use here with the A;,;’s playing the role of j1,..., Hn. 
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2 4 
which indeed implies that 1 — A > eivi — EN 
2 


To prove (3) sort the coordinates of v so that vı > v2 > --- > vp (with v; = 0 for i > n/2). 
Then 


n/2 n n/2 
2 2 
2 Asal (vj -v =y y As ¡(v = Vii) = So ail; lla 
i=1 j=1+1 i=1 


where c; denotes e. A; j. But c; is equal to the number of edges in G from the set {k : k < i} to 
its complement, divided by d. Hence, by the expansion of G, ci > pi, implying (using the fact that 


vi = 0 for i > 7/2): 


n/2 n/2 
2 Ais (vi — vj 220; vi) = $ (pivi— p- (¡—1)v7) = allyl, 
i=1 


establishing (3). W 


7.B.3 Error reduction using expanders. 


We now complete the analysis of the randomness efficient error reduction procedure described in 
Section 7.5. Recall, that this procedure was the following: let N = 2” where m is the number of 
coins the randomized algorithm uses. We use m+ O(k) random coins to select a k-vertex random 
walk in an expander graph Gy, and then output 1 if and only if the algorithm outputs 1 when given 
one of the vertices in the walk as random coins. To show this procedure works we need to show 
that if the probabilistic algorithm outputs 1 for at least half of the coins, then the probability that 
all the vertices of the walk correspond to coins on which the algorithm outputs 0 is exponentially 
small in k. This will be a direct consequence of the following theorem: (think of the set B below 
as the set of vertices corresponding to coins on which the algorithm outputs 0) 


THEOREM 7.38 (EXPANDER WALKS) 

Let G be an (N, d, A) graph, and let B C [N] be a set with |B| < GN. Let Xj,..., Xy 
be random variables denoting a k —1-step random walk from X 1, where X is chosen 
uniformly in [N]. Then, 


Pr[Vi<i<x Xi € B] MVE B+ AE 
(+) 


Note that if A and @ are both constants smaller than 1 then so is the expression (1— A) /B+A~. 
PROOF: For 1 <i < k, let B; be the event that X; € B. Note that the probability (*) we're trying 
to bound is Pr[B,] Pr[B2|B,]---Pr[B;|B1,..., By-1]. Let pê € RY be the vector representing 
the distribution of X;, conditioned on the events B,,...,B;. Denote by B the following linear 
transformation from R” to R”: for every u E RY, and l [N], (Bu); = u; if j € B and (Bu); = 0 
otherwise. It’s not hard to verify that p* = 1 (recall that 1 = (1/N,...,1/N) is the 
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vector representing the uniform distribution over [N]). Similarly, p? = ATAI Pri E rpg ABb 1 where 
A = A(G) is the adjacency matrix of G. Since every probability vector p satisfies |p|, = 1, 


(+) = |(BA)* “Bl, 
We bound this norm by showing that 
A nt > k-1 
(B4 Êi], < eve (4) 


which suffices since for every v € RN, |v|, < VNl|wIl, (see Note 7.24). 
To prove (4), we use the following definition and lemma: 


DEFINITION 7.39 (MATRIX NORM) 


If A is an m by n matrix, then [|4]| is the maximum number a such that || 4v||, < a||v||, for every 
vek”. 


Note that if A is a normalized adjacency matrix then ||A|| = 1 (as Al = 1 and ||Av||, < ||v]|, 
for every v). Also note that the matrix norm satisfies that for every two n by n matrices A, B, 
[A+ Bl| < ||Al] + ||B]| and ||ABl] < |All]. 


LEMMA 7.40 
Let A be a normalized adjacency matrix of an (n, d, A)-graph G. Let J be the adjacency matrix of 
the n-clique with self loops (i.e., Jij = 1/n for every i,j). Then 


A=(1-A)J+ AC (5) 
where ||C || < 1. 


Note that for every probability vector p, Jp is the uniform distribution, and so this lemma 

tells us that in some sense, we can think of a step on a (n,d, A)-graph as going to the uniform 
distribution with probability 1 — A, and to a different distribution with probability A. This is of 
course not completely accurate, as a step on a d-regular graph will only go the one of the d neighbors 
of the current vertex, but we'll see that for the purposes of our analysis, the condition (5) will be 
just as good.” 
PROOF OF LEMMA 7.40: Indeed, simply define C = 4(4— (1—A)J). We need to prove ||Cv||, < 
v||, for very v. Decompose v as v = u + w where u is a1 for some a and w L 1, and ||v||? = 
ull? + ||w||?. Since 41 = 1 and J1 = 1 we get that Cu = +(u — (1—A)u) = u. Now, let 
w = Aw. Then ||w’||, < A||w||, and, as we saw in the proof of Lemma 7.27, w’ 1 1. Furthermore, 
since the sum of the coordinates of w is zero, Jw = 0. We get that Cw = Fw”. Since w’ L u, 
Cowl? = ju + iwl? = llull? + iw’? < jul? + wl? = [por]. m 


Returning to the proof of Theorem 7.38, we can write BA = B((1 — A)J + AC), and hence 
BA||<(Q- A)118.3| + \||BC||. Since J’s output is always a vector of the form a1, ||BJ|| < yA. 
Also, because B is an operation that merely zeros out some parts of its input, ||Ê|| < 1 implying 


7Algebraically, the reason (5) is not equivalent to going to the uniform distribution in each step with probability 
1—A is that C is not necessarily a stochastic matrix, and may have negative entries. 
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||BC|| < 1. Thus, ||BAl] < (1-A)/8+A. Since B1 has the value 1/N in |B] places, ||B1||, = Yo, 
and hence ||(BA)*-!B1||, <(1- A VB + uE, establishing (4). W 

One can obtain a similar error reduction procedure for two-sided error algorithms by running 
the algorithm using the k sets of coins obtained from a k—1 step random walk and deciding on the 
output according to the majority of the values obtained. The analysis of this procedure is based 
on the following theorem, whose proof we omit: 


THEOREM 7.41 (EXPANDER CHERNOFF BOUND [?]) 

Let G be an (N, d, A)-graph and B C [N] with |B| = GN. Let X;,..., Xy be random 
variables denoting a k — 1-step random walk in G (where X is chosen uniformly). 
For every i € |k], define B; to be 1 if X; € B and 0 otherwise. Then, for every ô > 0, 


Pr RE: Bi 81> él < 9e NS? k/60 
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Chapter 8 


Interactive proofs 


“What is intuitively required from a theorem-proving procedure? First, that it is 
possible to “prove” a true theorem. Second, that it is impossible to “prove” a false 
theorem. Third, that communicating the proof should be efficient, in the following 
sense. It does not matter how long must the prover compute during the proving 
process, but it is essential that the computation required from the verifier is easy.” 
Goldwasser, Micali, Rackoff 1985 


The standard notion of a mathematical proof follows the certificate definition of NP. That is, 
to prove that a statement is true one provides a sequence of symbols that can be written down in a 
book or on paper, and a valid sequence exists only for true statements. However, people often use 
more general ways to convince one another of the validity of statements: they interact with one 
another, with the person verifying the proof (henceforth the verifier) asking the person providing 
it (henceforth the prover) for a series of explanations before he is convinced. 

It seems natural to try to understand the power of such interactive proofs from the complexity- 
theoretic perspective. For example, can one prove that a given formula is not satisfiable? (recall 
that is this problem is coNP-complete, it’s not believed to have a polynomial-sized certificate). 
The surprising answer is yes. Indeed, interactive proofs turned out to have unexpected powers 
and applications. Beyond their philosophical appeal, interactive proofs led to fundamental insights 
in cryptographic protocols, the power of approximation algorithms, program checking, and the 
hardness of famous “elusive” problems (i.e., NP-problems not known to be in P nor to be NP- 
complete) such as graph isomorphism and approximate shortest lattice vector. 


8.1 Warmup: Interactive proofs with a deterministic verifier 


Let us consider what happens when we introduce interaction into the NP scenario. That is, we’d 
have an interrogation-style proof system where rather than the prover send a written proof to the 
verifier, the prover and verifier interact with the verifier asking questions and the prover responding, 
where at the end the verifier decides whether or not to accept the input. Of course, both verifier 
and prover can keep state during the interaction, or equivalently, the message a party sends at any 
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point in the interaction can be a function of all messages sent and received so far. Formally, we 
make the following definition: 


DEFINITION 8.1 (INTERACTION OF DETERMINISTIC FUNCTIONS) 
Let f,g : (0,1)* — {0,1}* be functions. A k-round interaction of f and g on input x € {0,1}*, 
denoted by (f,g)(x) is the sequence of the following strings a,,...,a, € {0,1}* defined as follows: 


a, = f(x) 

a2 = g(x, a1) (1) 
4241 = f(z, UL)... , 25) 
0242 = g(z, Ql)...> a2i+1) 


(Where we consider a suitable encoding of i-tuples of strings to strings.) 
The output of f (resp. g) at the end of the interaction denoted outs (f,g)(x) (resp. outg( f, g)(x) 
) is defined to be f(x, a1,...,a%) (resp. g(a, @1,.--,@%)). 


DEFINITION 8.2 (DETERMINISTIC PROOF SYSTEMS) 
We say that a language L has a k-round deterministic interactive proof system if there's a deter- 


ministic TM V that on input x,aj,...,a¿ runs in time polynomial in |x|, satisfying: 
(Completeness)x € L => JP : {0,1}* — {0,1}* outy (V, P)(x) = 1 
(Soundness)a ¢ L => VP : {0,1}* — {0,1}* outy (V, P)(x) =1 


The class dIP contains all languages with a k(n)-round deterministic interactive proof systems 
with k(n) polynomial in n. 


It turns out this actually does not change the class of languages we can prove: 


THEOREM 8.3 
dIP = NP. 


PROOF: Clearly, every NP language has a l-round proof system. Now we prove that if a L has 
an interactive proof system of this type then L € NP. The certificate for membership is just the 


transcript (a1, a@2,...,a@,%) causing the verifier to accept. To verify this transcript, check that indeed 
V(x) = a1, V(@, a1, a2) = a3, ..., and V(x,a1,...,a,) = 1. If x € L then there indeed exists such 
a transcript. If there exists such a transcript (a1, ...,ap) then we can define a prover function P to 


satisfy P(x,a1) = a2, P(x, a1, a2,a3) = a4, etc. We see that outy (V, P)(x) = 1 and hence x € L. 
a 
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In order to realize the full potential of interaction, we need to let the verifier be probabilistic. The 
idea is that, similar to probabilistic algorithms, the verifier will be allowed to come to a wrong 
conclusion (e.g., accept a proof for a wrong statement) with some small probability. However, as in 
the case of probabilistic algorithms, this probability is over the verifier’s coins and the verifier will 
reject proofs for a wrong statement with good probability regardless of the strategy the prover uses. 
It turns out that the combination of interaction and randomization has a huge effect: as we will 
see, the set of languages which have interactive proof systems now jumps from NP to PSPACE. 


EXAMPLE 8.4 

As an example for a probabilistic interactive proof system, consider the following scenario: Marla 
claims to Arthur that she can distinguish between the taste of Coke (Coca-Cola) and Pepsi. To 
verify this statement, Marla and Arthur repeat the following experiment 50 times: Marla turns her 
back to Arthur, as he places Coke in one unmarked cup and Pepsi in another, choosing randomly 
whether Coke will be in the cup on the left or on the right. Then Marla tastes both cups and states 
which one contained which drinks. While, regardless of her tasting abilities, Marla can answer 
correctly with probability 5 by a random guess, if she manages to answer correctly for all the 50 
repetitions, Arthur can indeed be convinced that she can tell apart Pepsi and Coke. 


To formally define this we extend the notion of interaction to probabilistic functions (actually, 
we only need to do so for the verifier). To model an interaction between f and g where f is 
probabilistic, we add an additional m-bit input r to the function f in (1), that is having a; = f(x,r), 
az = f(x,r,a1,a2), etc. The interaction (f,g)(x) is now a random variable over r Er ([0,1)”. 
Similarly the output out s(f,g)(x) is also a random variable. 


DEFINITION 8.5 (IP) 

Let k : N — N be some function with k(n) computable in poly(n) time. A language L is in IP[k] 
if there is a Turing machine V such that on inputs x,r,aj,...,a¿, V runs in time polynomial in |x| 
and such that 


(Completeness) x E€ L > JP Pr[outy (V, Pix) = 
(Soundness) ge L => VP Pr[outy (V, P) (£) = 1] < 1/3. (3) 


We define IP = Ue>1 IP [n9]. 


REMARK 8.6 
The following observations on the class IP are left as an exercise (Exercise 1). 


1. Allowing the prover to be probabilistic (i.e., the answer function a; depends upon some 
random string used by the prover) does not change the class IP. The reason is that for 
any language L, if a probabilistic prover P results in making verifier V accept with some 
probability, then averaging implies there is a deterministic prover which makes V accept with 
the same probability. 
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Figure 8.1: Two isomorphic graphs. 


2. Since the prover can use an arbitrary function, it can in principle use unbounded computa- 
tional power (or even compute undecidable functions). However, one can show that given any 
verifier V, we can compute the optimum prover (which, given x, maximizes the verifier’s ac- 


ceptance probability) using poly(|x|) space (and hence 2poly (el) time). Thus IP € PSPACE. 


3. The probabilities of correctly classifying an input can be made arbitrarily close to 1 by using 
the same boosting technique we used for BPP (see Section ??): to replace 2/3 by 1—exp(—m), 
sequentially repeat the protocol m times and take the majority answer. In fact, using a more 
complicated proof, it can be shown that we can decrease the probability without increasing the 
number of rounds using parallel repetition (i.e., the prover and verifier will run m executions 
of the protocol in parallel). We note that the proof is easier for the case of public coin proofs, 
which will be defined below. 


4. Replacing the constant 2/3 in the completeness requirement (2) by 1 does not change the 
class IP. This is a nontrivial fact. It was originally proved in a complicated way but today 
can be proved using our characterization of IP later in Section 8.5. 


5. In contrast replacing the constant 2/3 by 1 in the soundness condition (3) is equivalent to 
having a deterministic verifier and hence reduces the class IP to NP. 


6. We emphasize that the prover functions do not depend upon the verifier’s random strings, 
but only on the messages/questions the verifier sends. In other words, the verifier’s random 
string is private. (Often these are called private coin interactive proofs.) Later we will also 
consider the model where all the verifier’s questions are simply obtained by tossing coins and 
revealing them to the prover (this is known as public coins or Arthur-Merlin proofs). 


8.3 Proving that graphs are not isomorphic. 


We'll now see an example of a language in IP that is not known to be in NP. Recall that the usual 
ways of representing graphs —adjacency lists, adjacency matrices— involve a numbering of the 
vertices. We say two graphs G and G2 are isomorphic if they are the same up to a renumbering 
of vertices. In other words, if there is a permutation 7 of the labels of the nodes of Gi such that 
(G1) = G2. The graphs in figure ??, for example, are isomorphic with m = (12)(3654). (That is, 
1 and 2 are mapped to each other, 3 to 6, 6 to 5, 5 to 4 and 4 to 1.) If Gi and Ga are isomorphic, 
we write Gi = G2. The Gl problem is the following: given two graphs G1, Ga (say in adjacency 
matrix representation) decide if they are isomorphic. Note that clearly Gl € NP, since a certificate 
is simply the description of the permutation 7. 

The graph isomorphism problem is important in a variety of fields and has a rich history (see 
[?]). Along with the factoring problem, it is the most famous NP-problem that is not known to be 
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either in P or NP-complete. The results of this section show that Gl is unlikely to be NP-complete, 
unless the polynomial hierarchy collapses. This will follow from the existence of the following proof 
system for the complement of Gl: the problem GNI of deciding whether two given graphs are not 
isomorphic. 


Protocol: Private-coin Graph Non-isomorphism 


V: pick ¿ € {1,2} uniformly randomly. Randomly permute the vertices of G; to get a 
new graph H. Send H to P. 


P: identify which of G1, G2 was used to produce H. Let G; be that graph. Send j to V. 


V: accept if i = j; reject otherwise. 


To see that Definition 8.5 is satisfied by the above protocol, note that if G; Æ Ga then there exists 
a prover such that Pr[V accepts] = 1, because if the graphs are non-isomorphic, an all-powerful 
prover can certainly tell which one of the two is isomorphic to H. On the other hand, if Gi = Ga 
the best any prover can do is to randomly guess, because a random permutation of G1 looks exactly 
like a random permutation of Gə. Thus in this case for every prover, Pr[Vaccepts] < 1/2. This 
probability can be reduced to 1/3 by sequential or parallel repetition. 


8.4 Public coins and AM 


Allowing the prover full access to the verifier’s random string leads to the model of interactive 
proofs with public-coins. 


DEFINITION 8.7 (AM, MA) 

For every k we denote by AMTk] the class of languages that can be decided by a k round interactive 
proof in which each verifier’s message consists of sending a random string of polynomial length, 
and these messages comprise of all the coins tossed by the verifier. A proof of this form is called a 
public coin proof (it is sometimes also known an Arthur Merlin proof).! 

We define by AM the class AM[2].? That is, AM is the class of languages with an interactive 
proof that consist of the verifier sending a random string, the prover responding with a message, 
and where the decision to accept is obtained by applying a deterministic polynomial-time function 
to the transcript. The class MA denotes the class of languages with 2-round public coins interactive 
proof with the prover sending the first message. That is, L € MA if there's a proof system for L 
that consists of the prover first sending a message, and then the verifier tossing coins and applying 
a polynomial-time predicate to the input, the prover’s message and the coins. 


Arthur was a famous king of medieval England and Merlin was his court magician. Babai named these classes 
by drawing an analogy between the prover’s infinite power and Merlin’s magic. One “justification” for this model 
is that while Merlin cannot predict the coins that Arthur will toss in the future, Arthur has no way of hiding from 
Merlin’s magic the results of the coins he tossed in the past. 

“Note that AM = AM[2] while IP = IP[poly]. While this is indeed somewhat inconsistent, this is the standard 
notation used in the literature. We note that some sources denote the class AM[3] by AMA, the class AM[4] by 
AMAM etc. 
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Note that clearly for every k, AM[k] C IP[k]. The interactive proof for GNI seemed to crucially 
depend upon the fact that P cannot see the random bits of V. If P knew those bits, P would know 
i and so could trivially always guess correctly. Thus it may seem that allowing the verifier to keep 
its coins private adds significant power to interactive proofs, and so the following result should be 
quite surprising: 


THEOREM 8.8 ([GS87]) 
For every k : N — N with k(n) computable in poly(n), 


IP[k] € AM[k +2] 


The central idea of the proof of Theorem 8.8 can be gleaned from the proof for the special case 
of GNI. 


THEOREM 8.9 
GNI € AM[k] for some constant k > 2. 


The key idea in the proof of Theorem 8.9 is to look at graph nonisomorphism in a different, 
more quantitative, way. (Aside: This is a good example of how nontrivial interactive proofs can be 
designed by recasting the problem.) Consider the set S = {H : H = G or H = G2}. Note that it 
is easy to prove that a graph H is a member of S, by providing the permutation mapping either 
G or Ga to H. The size of this set depends on whether G4 is isomorphic to G2. An n vertex graph 
G has at most n! equivalent graphs. If Gi and G2 have each exactly n! equivalent graphs (this will 
happen if for i = 1,2 there's no non-identity permutation 7 such that 7(G;) = G;) we'll have that 


if G1 % Ga then |S| = 2n! (4) 
if G1 = Ga then |S| =n! (5) 


(To handle the general case that G1 or Gz may have less than n! equivalent graphs, we actually 
change the definition of S to 


S={(H,7):H=G, or H = Gə and 7 € aut(H)} 


where 7 € aut(H) if 7(H) = H. It is clearly easy to prove membership in the set S and it can be 
verified that S satisfies (4) and (5).) 

Thus to convince the verifier that G1 % G2, the prover has to convince the verifier that case (4) 
holds rather than (5). This is done by using a set lower bound protocol. 


8.4.1 Set Lower Bound Protocol. 


In a set lower bound protocol, the prover proves to the verifier that a given set S (where membership 
in S is efficiently verifiable) has cardinality at least K up to accuracy of, say, factor of 2. That 
is, if |S| > K then the prover can cause the verifier to accept with high probability, while if 
|S| < K/2 then the verifier will reject with high probability, no matter what the prover does. By 
the observations above, such a protocol suffices to complete the proof of Theorem 8.9. 
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Tool: Pairwise independent hash functions. 


The main tool we use for the set lower bound protocol is a pairwise independent hash function 
collection. This is a simple but incredibly useful tool that has found numerous applications in 
complexity theory and computer science at large (see Note 8.13). 


DEFINITION 8.10 (PAIRWISE INDEPENDENT HASH FUNCTIONS) 

Let H,,x be a collection of functions from {0,1}" to {0, 1}*. We say that Hn,k is 
pairwise independent if for every x,x’ € {0,1}" with x Æ 2’ and for every y,y' € 
{0, i PrherHn lhl) =y A h(a’) = y] = g-m 


Note that an equivalent formulation is that for every two distinct strings z,2’ € {0,1}” the 
random variable (h(x),h(a’)) for h chosen at random from H»,x is distributed according to the 
uniform distribution on {0,1}* x {0,1}*. 

Recall that we can identify the elements of (0,1)” with the finite field (see Section A.4 in 
the appendix), denoted GF(2”), containing 2” elements, whose addition (+) and multiplication (-) 
operations satisfy the usual commutative and distributive laws, where and every element x has an 
additive inverse (denoted by —x) and, if nonzero, a multiplicative inverse (denoted by x~!). The 
following theorem provides a construction of an efficiently computable pairwise independent hash 
functions (see also Exercise 4 for a different construction): 


THEOREM 8.11 (EFFICIENT PAIRWISE INDEPENDENT HASH FUNCTIONS) 

For every n define the collection Hn,» to be {ha,b}a,beGr(2n) Where for every a,b € GF(2”), the func- 
tion hap : GF(2") > GF(2”) maps x to ax +b. Then, Hn» is a collection of pairwise independent 
hash functions. 


REMARK 8.12 

Theorem 8.11 implies the existence of an efficiently computable pairwise independent hash functions 
Hn,k for every n,k: if k > mn we can use the collection Hk, and reduce the size of the input to n 
by padding it with zeros. If k < n then we can use the collection H,,n and truncate the last n — k 
bits of the output. 


PROOF: For every z # x’ € GF(2”) and y, y' € GE(2”), hap(x) = y and hay (2) = y’ iff a,b satisfy 
the equations: 


a-x+b=y 
a: +b=y 


These imply a - (x — 2’) = y — y! or a= (y — y')(x — x')?. Since b = y — a: x, the pair (a,b) is 
completely determined by these equations, and so the probability that this happens over the choice 
of a,b is exactly one over the number of possible pairs, which indeed equals sin a 
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NOTE 8.13 (THE HASHING PARADIGM) 

A hash function collection is a collection of functions mapping a large uni- 
verse, say (0, 1)”, to a smaller universe, say {0,1}* for k < n. Typically, we 
require of such a collection that it maps its input in a fairly uniform way to 
the output range. For example, if S is a subset of (0, 1)” then we wish that, 
if h is chosen at random from the collection, then most elements of {0, 1)" 
have roughly |S|2~-* preimages in S (which is the expected number if h was 
a completely random function). In particular, if S has size roughly 2° then 
we expect the mapping to be one-to-one or almost one-to-one, and so there 
should be a relatively small number of collisions: pairs x 4 x’ € S such that 
h(x) = h(x’). Therefore, the image of S under h should look like this: 


{0,1}" 


{0,1} 


PAS 


In databases, hash functions are used to maintain very efficient databases 
(that allow fast membership queries to a subset S C (0,1)” of size 2* re- 
quiring only 2" as opposed to 2” bits of storage). In theoretical computer 
science, hash functions have a variety of uses. An example is Lemma 9.16 
of the next chapter that shows that if the collection is pairwise independent 
and S C (0,1)” has size roughly 2", then with good probability the value 
0* will have exactly one preimage in S. 

In all these cases it is important that the hash function is chosen at random 
from some collection independently of the choice of set S. It is easy to see 
that if k is small enough (e.g., k < n/2) then for every h : {0,1}" — {0,1}* 
there is a set S C {0,1}" of size 2% that is “very bad” for h in the sense that 
all the members of S map to the same element under h. 

Pairwise independent hash functions are but one example of a hash func- 
tion collection. Several types of such collections are known in the literature 
featuring various tradeoffs between efficiency and uniformity of output. 
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The lower-bound protocol. 


The lower-bound protocol is as follows: 


Protocol: Goldwasser-Sipser Set Lowerbound 


Conditions: S C {0,1}' is a set such that membership in S can be certified. Both 
parties know a number K. The prover's goal is to convince the verifier that |S| > K 


and the verifier should reject if |S| < E Let k be a number such that 2472 < K < 
DETA, 


V: Randomly pick a function h : {0,1} — {0,1}* from a pairwise independent hash 
function collection Hm,- Pick y Ep {0, 1). Send h, y to prover. 


P: Try to find an x € S such that h(x) = y. Send such an g to V, together with a 
certificate that x € S. 


V’s output: If certificate validates that x € S and h(x) = y, accept; otherwise reject. 


Let p= 5. If |S| < K then clearly |h(S)| < p and so the verifier will accept with probability 
at most $. The main challenge is to show that if |S| > K then the verifier will accept with 
probability noticeably larger than p/2 (the gap between the probabilities can then be amplified 
using repetition). That is, it suffices to prove 


CLAIM 8.13.1 A 
Let S C [0,1)” satisfy |5| < 4. Then, 


yu 


Pr [Avesh(x) = y] > í 
hERHm,kYER{0,1}" 


PROOF: For every y € {0,1}”, we’ll prove the claim by showing that 


Pr  [Breshlz)= y] > 2p, 
ae TES ( ) y] ZP 


(where p = |5|/2*). Indeed, for every x € S define the event E, to hold if h(x) = y. Then, 
Prl3xesh(x) = y] = Pr[Ures Er] but by the inclusion-exclusion principle this is at least 


S > Pr[Ex]- 3 Y PriEsn E; 


TES xx Es 


= 272k 


However, by pairwise independence, if x 4 2’, then Pr[E,] = 27% and Pr[E, A E] and so 


this probability is at least 
1 2 
[S| |S| |S| (1 |S| ) $ žy 


2k 2 9k — 9k 2k+1 
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Figure 8.2: AMIK] looks like []?, with the Y quantifier replaced by probabilitic choice. 


Proving Theorem 8.9. The public-coin interactive proof system for GNI consists of the verifier 
and prover running several iterations of the set lower bound protocol for the set S as defined 
above, where the verifier accepts iff the fraction of accepting iterations was at least 0.6p (note that 
both parties can compute p). Using the Chernoff bound (Theorem A.18) it can be easily seen 
that a constant number of iteration will suffices to ensure completeness probability at least 3 and 


soundness error at most 3. A 


REMARK 8.14 

How does this protocol relate to the private coin protocol of Section 8.3? The set S roughly 
corresponds to the set of possible messages sent by the verifier in the protocol, where the verifier’s 
message is a random element in S. If the two graphs are isomorphic then the verifier’s message 
completely hides its choice of a random i €r {1,2}, while if they're not then it completely reveals it 
(at least to a prover that has unbounded computation time). Thus roughly speaking in the former 
case the mapping from the verifier’s coins to the message is 2-to-1 while in the latter case it is 
1-to-1, resulting in a set that is twice as large. Indeed we can view the prover in the public coin 
protocol as convincing the verifier that its probability of convincing the private coin verifier is large. 
While there are several additional intricacies to handle, this is the idea behind the generalization 
of this proof to show that IP[k] € AM[k + 2]. 


REMARK 8.15 

Note that, unlike the private coins protocol, the public coins protocol of Theorem 8.9 does not enjoy 
perfect completeness, since the set lowerbound protocol does not satisfy this property. However, 
we can construct a perfectly complete public-coins set lowerbound protocol (see Exercise 3), thus 
implying a perfectly complete public coins proof for GNI. Again, this can be generalized to show that 
any private-coins proof system (even one not satisfying perfect completeness) can be transformed 
into a perfectly complete public coins system with a similar number of rounds. 


8.4.2 Some properties of IP and AM 
We state the following properties of IP and AM without proof: 


1. (Exercise 5) AM[2] = BP -NP where BP - NP is the class in Definition ??. In particular it 
follows that AM[2] C XB. 


2. (Exercise 4) For constants k > 2 we have AM[k] = AM[2]. This “collapse” is somewhat 
surprising because AM/|k] at first glance seems similar to PH with the V quantifiers changed 
to “probabilistic V” quantifiers, where most of the branches lead to acceptance. See Figure 8.2. 


3. It is open whether there is any nice characterization of AMÍ[o(n)], where o(n) is a suitably 
slow growing function of n, such as log logn. 
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8.4.3 Can Gl be NP-complete? 


We now prove that if Gl is NP-complete then the polynomial hierarchy collapses. 
THEOREM 8.16 ([?]) 
If Gl is NP-complete then Xa = Te. 


PROOF: If Gl is NP-complete then GNI is coNP-complete which implies that there exists a function 
f such that for every n variable formula y, Vyp(y) holds iff f(y) € GNI. Let 


Y = dreo,1}"Vye{o,1}" (T, y) 


be a 225AT formula. We have that y is equivalent to 


Jze{0,1}" g(x) € GNI 


where g(x) = f(Pyo). 

Using Remark 8.15 and the comments of Section 8.4.2, we have that GNI has a two round AM 
proof with perfect completeness and (after appropriate amplification) soundness error less than 
27”, Let V be the verifier algorithm for this proof system, and denote by m the length of the 
verifier’s random tape and by m’ the length of the prover’s message and . We claim that 1 is 
equivalent to 


V* = Vreto} see {0,1}" Fac {o,17" V (gle), r,a) = 1 


Indeed, by perfect completeness if Y is satisfiable then y* is satisfiable. If Y is not satisfiable 
then by the fact that the soundness error is at most 27”, we have that there exists a single string 
r € {0,1}” such that for every x with g(x) ¢ GNI, there's no a such that V(g(x),r, a) = 1, and so 
w* is not satisfiable. Since w* can easily be reduced to a IL2SAT formula, we get that Ne C IIe, 
implying (since M2 = coll2) that Xə = Il>. W 


8.5 IP = PSPACE 


In this section we show a surprising characterization of the set of languages that have interactive 
proofs. 


THEOREM 8.17 (LFKN, SHAMIR, 1990) 
IP = PSPACE. 


Note that this is indeed quite surprising: we already saw that interaction alone does not increase 
the languages we can prove beyond NP, and we tend to think of randomization as not adding 
significant power to computation (e.g., we'll see in Chapter 16 that under reasonable conjectures, 
BPP =P). As noted in Section 8.4.2, we even know that languages with constant round interactive 
proofs have a two round public coins proof, and are in particular contained in the polynomial 
hierarchy, which is believed to be a proper subset of PSPACE. Nonetheless, it turns out that the 
combination of sufficient interaction and randomness is quite powerful. 
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By our earlier Remark 8.6 we need only show the direction PSPACE C IP. To do so, we’ll show 
that TQBF € IP[poly(n)]. This is sufficient because every L € PSPACE is polytime reducible to 
TQBF. We note that our protocol for TQBF will use public coins and also has the property that if 
the input is in TQBF then there is a prover which makes the verifier accept with probability 1. 

Rather than tackle the job of designing a protocol for TQBF right away, let us first think about 
how to design one for 3SAT. How can the prover convince the verifier than a given 3CNF formula 
has no satisfying assignment? We show how to prove something even more general: the prover can 
prove to the verifier what the number of satisfying assignments is. (In other words, we will design 
a prover for #SAT.) The idea of arithmetization introduced in this proof will also prove useful in 
our protocol for TQBF. 


8.5.1 Arithmetization 


The key idea will be to take an algebraic view of boolean formulae by representing them as polyno- 
mials. Note that 0, 1 can be thought of both as truth values and as elements of some finite field F. 
Thus we have the following correspondence between formulas and polynomials when the variables 
take 0/1 values: 


Ay — X- Y 

=ar —> 1-X 

gVy == 1-(1-X)(1-Y) 
gVyVnz —= 1-(1-X)1-Y)Z 


Given any 3CNF formula p(11,12,..., £n) with m clauses, we can write such a degree 3 polyno- 
mial for each clause. Multiplying these polynomials we obtain a degree 3m multivariate polynomial 
P,(X1, X2,...,Xn) that evaluates to 1 for satisfying assignments and evaluates to 0 for unsatis- 
fying assignments. (Note: we represent such a polynomial as a multiplication of all the degree 3 
polynomials without “opening up” the parenthesis, and so P¿(X1, X2,..., Xn) has a representation 
of size O(m).) This conversion of y to P, is called arithmetization. Once we have written such 
a polynomial, nothing stops us from going ahead and and evaluating the polynomial when the 
variables take arbitrary values from the field F instead of just 0,1. As we will see, this gives the 
verifier unexpected power over the prover. 


8.5.2 Interactive protocol for #SATp 


To design a protocol for 3SAT we give a protocol for #SATp, which is a decision version of the 
counting problem #SAT we saw in Chapter ??: 


#SATp = {(¢, K) : K is the number of satisfying assignments of ¢}. 


and ¢ is a 3CNF formula of n variables and m clauses. 


THEOREM 8.18 
#SATp € IP. 
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PROOF: Given input (¢, K}, we construct, by arithmetization, Py. The number of satisfying as- 
signments #¢ of ¢ is: 


PSJ Jo we Y, Hürrem (6) 


b1€{0,1} b2€{0,1}  bne(0,1) 


To start, the prover sends to the verifier a prime p in the interval (2”, 227]. The verifier can check 
that p is prime using a probabilistic or deterministic primality testing algorithm. All computations 
described below are done in the field F = F, of numbers modulo p. Note that since the sum in (6) 
is between 0 and 2”, this equation is true over the integers iff it is true modulo p. Thus, from now 
on we consider (6) as an equation in the field F,. We'll prove the theorem by showing a general 
protocol, Sumcheck, for verifying equations such as (6). 


Sumcheck protocol. 


Given a degree d polynomial g(X;,..., Xn), an integer K, and a prime p, we present an interactive 


proof for the claim 
K= Y SO e+ Y g(X,...,Xn) (7) 


b1€10,1) b2€{0,1}  bnef0,1) 


(where all computations are modulo p). To execute the protocol V will need to be able to evaluate 
the polynomial g for any setting of values to the variables. Note that this clearly holds in the case 
g = Pg. 

For each sequence of values 62, b3,..., bn to X2, X3,..., Xn, note that g( X1, b2,b3,...,bn) is a 
univariate degree d polynomial in the variable X1. Thus the following is also a univariate degree d 
polynomial: 

W= J e Y gl Xi, bo... bn) 


b2€{0,1} bn €{0,1} 


If Claim (7) is true, then we have h(0) + h(1) = K. 
Consider the following protocol: 


Protocol: Sumcheck protocol to check claim (7) 


V: If n= 1 check that g(1) + g(0) = K. If so accept, otherwise reject. If n > 2, ask P 
to send h(X1) as defined above. 


P: Sends some polynomial s(X1) (if the prover is not “cheating” then we'll have s(X1) = 
h(X1)). 


V: Reject if s(0) + s(1) 4 K; otherwise pick a random a. Recursively use the same 
protocol to check that 


s(a) = y e. y g(a, b2,..., bn). 


be {0,1} bn€ {0,1} 
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If Claim (7) is true, the prover that always returns the correct polynomial will always convince 
V. If (7) is false then we prove that V rejects with high probability: 


n 
Pr[V rejects (K, g)] > (1 — 2) : (8) 
With our choice of p, the right hand side is about 1 — dn/p, which is very close to 1 since d < n° 
and p > nt. 

Assume that (7) is false. We prove (8) by induction on n. For n = 1, V simply evaluates 
g(0),g(1) and rejects with probability 1 if their sum is not K. Assume the hypothesis is true for 
degree d polynomials in n — 1 variables. 

In the first round, the prover P is supposed to return the polynomial h. If it indeed returns 
h then since h(0) + h(1) 4 K by assumption, V will immediately reject (i.e., with probability 1). 
So assume that the prover returns some s(X1) different from h(X¡). Since the degree d nonzero 
polynomial s(X,) — h(X1) has at most d roots, there are at most d values a such that s(a) = h(a). 
Thus when V picks a random a, 


Pr{s(a) 4 h(a)] > 1 - z, (9) 


If s(a) # h(a) then the prover is left with an incorrect claim to prove in the recursive step. 
By the induction hypothesis, the prover fails to prove this false claim with probability at least 


n—1 
> (1 — 4) . Thus we have 


Bay eads (1 z 2) (1 _ Y x (1 _ “\" (10) 


This finishes the induction. 
A 


8.5.3 Protocol for TQBF: proof of Theorem 8.17 


We use a very similar idea to obtain a protocol for TQBF. Given a quantified Boolean formula 
Y = J21Vx2323 :-*Vino(t1,..., Un), we use arithmetization to construct the polynomial Py. We 
have that Y € TQBF if and only if 


o< Y [J > = ][ Pstt0r,...,dn) (11) 


b1€10,1) b2€{0,1} b3€{0,1}  bnef0,1) 


A first thought is that we could use the same protocol as in the #SATp case, except check that 
s(0)-s(1) = K when you have a [[. But, alas, multiplication, unlike addition, increases the degree of 
the polynomial — after k steps, the degree could be 2*. Such polynomials may have 2* coefficients 
and cannot even be transmitted in polynomial time if k > log n. 

The solution is to look more closely at the polynomials that are are transmitted and their relation 
to the original formula. We'll change Y into a logically equivalent formula whose arithmetization 
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does not cause the degrees of the polynomials to be so large. The idea is similar to the way circuits 
are reduced to formulas in the Cook-Levin theorem: we’ll add auxiliary variables. Specifically, we’ll 
change y to an equivalent formula y” that is not in prenex form in the following way: work from 
right to left and whenever encountering a V quantifier on a variable x; — that is, when considering 
a postfix of the form V,,7(11,...,1;), where 7 may contain quantifiers over additional variables 
Ti+1;» ++, Ty — ensure that the variables x1,...,x¡ never appear to the right of another V quantifier 
in 7 by changing the postfix to Vx;3x+,...,1(1 = 21) A- A (a = xi) AT(£1,..., 2n). Continuing 
this way we'll obtain the formula y” which will have O(n?) variables and will be at most O(n?) 
larger than 4. It can be seen that the natural arithmetization for y’ will lead to the polynomials 
transmitted in the sumcheck protocol never having degree more than 2. 


Note that the prover needs to prove that the arithmetization of Y’ leads to a number K different 
than 0, but because of the multiplications this number can be as large as 2?”. Nevertheless the 
prover can find a prime p between 0 and 2” such that K mod p 4 0 (in fact as we saw in Chapter 7 
a random prime will do). This finishes the proof of Theorem 8.17. E 


REMARK 8.19 

An alternative way to obtain the same result (or, more accurately, an alternative way to describe 
the same protocol) is to notice that for x € {0,1}, x* = z for all k > 1. Thus, in principle we can 
convert any polynomial p(z1,..., £n) into a multilinear polynomial q(x1,..., £n) i.e., the degree of 
q(-) in any variable x; is at most one) that agrees with p(-) on all z1,..., £n € {0,1}. Specifically, 
for any polynomial p(-) let L;(p) be the polynomial defined as follows 


L;(p)(21, e Lin) = ei (zi; 0.39 GAS: 1, Li4+1, ek ,Zn)+ 
(1 = zi)P(z1, aoe ¡Bicis OF WE ne va) (12) 


then Lı (Lə(--- (Ln(p)--- ) is such a multilinear polynomial agreeing with p(-) on all values in (0,1). 
We can thus use O(n”) invocations operator to convert (11) into an equivalent form where all the 
intermediate polynomials sent in the sumcheck protocol are multilinear. We'll use this equivalent 
form to run the sumcheck protocol, where in addition to having round for a > or [| operator, 
we'll also have a round for each application of the operator L (in such rounds the prover will send 
a polynomial of degree at most 2). 


8.6 The power of the prover 


A curious feature of many known interactive proof systems is that in order to prove membership 
in language L, the prover needs to do more powerful computation than just deciding membership 
in L. We give some examples. 


1. The public coin system for graph nonisomorphism in Theorem 8.9 requires the prover to 
produce, for some randomly chosen hash function h and a random element y in the range of 
h, a graph H such that h(#) is isomorphic to either Gi or G2 and h(x) = y. This seems 
harder than just solving graph non-isomorphism. 
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2. The interactive proof for 3SAT, a language in coNP, requires the prover to do #P compu- 
tations, doing summations of exponentially many terms. (Recall that all of PH is in P*P.) 


In both cases, it is an open problem whether the protocol can be redesigned to use a weaker 
prover. 

Note that the protocol for TQBF is different in that the prover’s replies can be computed in 
PSPACE as well. This observation underlies the following result, which is in the same spirit 
as the Karp-Lipton results described in Chapter ??, except the conclusion is stronger since MA 
is contained in “2 (indeed, a perfectly complete MA-proof system for L trivially implies that 
LE X2). 


THEOREM 8.20 
If PSPACE C P/poly then PSPACE = MA. 


PROOF: If PSPACE C P/poly then the prover in our TQBF protocol can be replaced by a circuit 
of polynomial size. Merlin (the prover) can just give this circuit to Arthur (the verifier) in Round 
1, who then runs the interactive proof using this “prover.” No more interaction is needed. Note 
that there is no need for Arthur to put blind trust in Merlin’s circuit, since the correctness proof of 
the TQBF protocol shows that if the formula is not true, then no prover can make Arthur accept 
with high probability. W 


In fact, using the Karp-Lipton theorem one can prove a stronger statement, see Lemma ?? 
below. 


8.7 Program Checking 


The discovery of the interactive protocol for the permanent problem was triggered by a field called 
program checking. Blum and Kannan’s motivation for introducing this field was the fact that 
program verification (deciding whether or not a given program solves a certain computational task) 
is undecidable. They observed that in many cases we can guarantee a weaker guarantee of the 
program’s “correctness” on an instance by instance basis. This is encapsulated in the notion of 
a program checker. A checker C for a program P is itself another program that may run P as 
a subroutine. Whenever P is run on an input x, C’s job is to detect if P’s answer is incorrect 
(“buggy”) on that particular instance x. To do this, the checker may also compute P’s answer on 
some other inputs. Program checking is sometimes also called instance checking, perhaps a more 
accurate name, since the fact that the checker did not detect a bug does not mean that P is a 
correct program in general, but only that P’s answer on x is correct. 


DEFINITION 8.21 
Let P be a claimed program for computational task T. A checker for T is a probabilistic polynomial 
time TM, C, that, given any x, has the following behavior: 


1. If Pis a correct program for T (i.e., Vy P(y) = T(y)), then P[C” accepts P(x)| > 3 


2. If P(x) # T(x) then P[C” accepts P(x)] < 3 
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Note that in the case that P is correct on x (i.e., P(x) = C(x)) but the program P is not correct 
everywhere, there is no guarantee on the output of the checker. 

Surprisingly, for many problems, checking seems easier than actually computing the problem. 
(Blum and Kannan’s suggestion was to build checkers into the software whenever this is true; the 
overhead introduced by the checker would be negligible.) 


EXAMPLE 8.22 (CHECKER FOR GRAPH NON-ISOMORPHISM) 
The input for the problem of Graph Non-Isomorphism is a pair of labelled graphs (G1, Ga), and 
the problem is to decide whether G1 = G2. As noted, we do not know of an efficient algorithm for 
this problem. But it has an efficient checker. 

There are two types of inputs, depending upon whether or not the program claims Gi = Go. 
If it claims that Gi = Ga then one can change the graph little by little and use the program to 
actually obtain the permutation m (). We now show how to check the claim that G1 # Ga using 
our earlier interactive proof of Graph non-isomorphism. 

Recall the IP for Graph Non-Isomorphism: 


e In case prover admits G1 Æ Gə repeat k times: 
e Choose i Er {1,2}. Permute G; randomly into H 
e Ask the prover (G1, H); (G2, H) and check to see if the prover’s first answer is consistent. 


Given a computer program that supposedly computes graph isomorphism, P, how would we check 
its correctness? The program checking approach suggests to use an IP while regarding the program 
as the prover. Let C be a program that performs the above protocol with P as the prover, then: 


THEOREM 8.23 
If P is a correct program for Graph Non-Isomorphism then C outputs ”correct” always. Otherwise, 
if P(G1, G2) is incorrect then P[C outputs ”correct” ] < 27". Moreover, C runs in polynomial time. 


8.7.1 Languages that have checkers 


Whenever a language L has an interactive proof system where the prover can be implemented 
using oracle access to L, this implies that L has a checker. Thus, the following theorem is a direct 
consequence of the interactive proofs we have seen: 


‘THEOREM 8.24 
The problems Graph Isomorphism (Gl), Permanent (perm) and True Quantified Boolean Formulae 
(TQBF) have checkers. 


Using the fact that P-complete languages are reducible to each other via NC-reductions, it suffices 
to show a checker in NC for one P-complete language (as was shown by Blum & Kannan) to obtain 
the following interesting fact: 
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THEOREM 8.25 
For any P-complete language there exists a program checker in NC 


Since we believe that P-complete languages cannot be computed in NC, this provides additional 
evidence that checking is easier than actual computation. 


8.8 Multiprover interactive proofs (MIP) 


It is also possible to define interactive proofs that involve more than one prover. The important 
assumption is that the provers do not communicate with each other during the protocol. They 
may communicate before the protocol starts, and in particular, agree upon a shared strategy for 
answering questions. (The analogy often given is that of the police interrogating two suspects in 
separate rooms. The suspects may be accomplices who have decided upon a common story to tell 
the police, but since they are interrogated separately they may inadvertently reveal an inconsistency 
in the story.) 

The set of languages with multiprover interactive provers is call MIP. The formal definition is 
analogous to Definition 8.5. We assume there are two provers (though one can also study the case 
of polynomially many provers; see the exercises), and in each round the verifier sends a query to 
each of them —the two queries need not be the same. Each prover sends a response in each round. 

Clearly, IP C MIP since we can always simply ignore one prover. However,it turns out that 
MIP is probably strictly larger than IP (unless PSPACE = NEXP). That is, we have: 


THEOREM 8.26 ([BFL91]) 
NEXP = MIP 


We will outline a proof of this theorem in Chapter ??. One thing that we can do using two 
rounds is to force non-adaptivity. That is, consider the interactive proof as an “interrogation” 
where the verifier asks questions and gets back answers from the prover. If the verifier wants to 
ensure that the answer of a prover to the question q is a function only of q and does not depend 
on the previous questions the prover heard, the prover can ask the second prover the question q 
and accept only if both answers agree with one another. This technique was used to show that 
multi-prover interactive proofs can be used to implement (and in fact are equivalent to) a model 
of a “probabilistically checkable proof in the sky”. In this model we go back to an NP-like notion 
of a proof as a static string, but this string may be huge and so is best thought of as a huge table, 
consisting of the prover’s answers to all the possible verifier’s questions. The verifier checks the 
proof by looking at only a few entries in this table, that are chosen randomly from some distribution. 
If we let the class PCP|r, q] be the set of languages that can be proven using a table of size 2” and 
q queries to this table then Theorem 8.26 can be restated as 


THEOREM 8.27 (THEOREM 8.26, RESTATED) 
NEXP = PCP|poly, poly] = U¿PCP[n*, n°] 


It turns out Theorem 8.26 can be scaled down to to obtain NP = PCP{polylog, polylog]. In 
fact (with a lot of work) the following is known: 
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THEOREM 8.28 (THE PCP THEOREM, [AS98, ALM798}) 
NP = PCP|[O(logn), O(1)] 


This theorem, which will be proven in Chapter 18, has had many applications in complexity, 
and in particular establishing that for many NP-complete optimization problems, obtaining an 
approximately optimal solution is as hard as coming up with the optimal solution itself. Thus, it 
seems that complexity theory has gone a full circle with interactive proofs: by adding interaction, 
randomization, and multiple provers, and getting to classes as high as NEXP, we have gained new 
and fundamental insights on the class NP the represents static deterministic proofs (or equivalently, 
efficiently verifiable search problems). 


WHAT HAVE WE LEARNED? 


e An interactive proof is a generalization of mathematical proofs in which the 
prover and polynomial-time probabilistic verifier interact. 


e Allowing randomization and interaction seems to add significantly more power 
to proof system: the class IP of languages provable by a polynomial-time 
interactive proofs is equal to PSPACE. 


e All languages provable by a constant round proof system are in the class AM: 
that is, they have a proof system consisting of the the verifier sending a single 
random string to the prover, and the prover responding with a single message. 


Chapter notes and history 


Interactive proofs were defined in 1985 by Goldwasser, Micali, Rackoff [GMR89] for cryptographic 
applications and (independently, and using the public coin definition) by Babai and Moran [BM88]. 
The private coins interactive proof for graph non-isomorphism was given by Goldreich, Micali and 
Wigderson [GMW87]. Simulations of private coins by public coins we given by Goldwasser and 
Sipser [GS87]. The general feeling at the time was that interactive proofs are only a “slight” 
extension of NP and that not even 3SAT has interactive proofs. The result IP = PSPACE was 
a big surprise, and the story of its discovery is very interesting. 


In the late 1980s, Blum and Kannan [BK95] introduced the notion of program checking. Around 
the same time, manuscripts of Beaver and Feigenbaum [BF 90] and Lipton [Lip91] appeared. In- 
spired by some of these developments, Nisan proved in December 1989 that #SAT has multiprover 
interactive proofs. He announced his proof in an email to several colleagues and then left on va- 
cation to South America. This email motivated a flurry of activity in research groups around the 
world. Lund, Fortnow, Karloff showed that #SAT is in IP (they added Nisan as a coauthor and the 
final paper is [LFK92]). Then Shamir showed that IP =PSPACE [Sha92] and Babai, Fortnow and 
Lund [BFL91] showed MIP = NEXP. The entire story —as well as related developments—are 
described in Babai’s entertaining survey [Bab90]. 

Vadhan [Vad00] explores some questions related to the power of the prover. 
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The result that approximating the shortest vector is probably not NP-hard (as mentioned in 
the introduction) is due to Goldreich and Goldwasser [GG00]. 


Exercises 


$1 Prove the assertions in Remark 8.6. That is, prove: 


(a) Let IP’ denote the class obtained by allowing the prover to be probabilistic in Defini- 
tion 8.5. That is, the prover’s strategy can be chosen at random from some distribution 
on functions. Prove that IP’ = IP. 


(b) Prove that IP C PSPACE. 


(c) Let IP’ denote the class obtained by changing the constant 2/3 in (2) and (3) to 1—27!#!, 
Prove that IP’ = IP. 


(d) Let IP’ denote the class obtained by changing the constant 2/3 in (2) to 1. Prove that 


IP’ = IP. 
(e) Let IP’ denote the class obtained by changing the constant 2/3 in (3) to 1. Prove that 
IP’ = NP. 


$2 We say integer y is a quadratic residue modulo m if there is an integer x such that y = x? 


(mod m). Show that the following language is in IP[2]: 


QNR = {(y,m): y is not a quadratic residue modulo m}. 


83 Prove that there exists a perfectly complete AM[O(1)] protocol for the proving a lowerbound 
on set size. 
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§4 Prove that for every constant k > 2, AM[k +1] C AMIK]. 
85 Show that AM[|2] = BP - NP 
86 [BFNW93] Show that if EXP C P/poly then EXP = MA. 
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87 


88 
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Show that the problem Gl is downward self reducible. That is, prove that given two graphs 
G1,G2 on n vertices and access to a subroutine P that solves the Gl problem on graphs with 
up to n — 1 vertices, we can decide whether or not Gi and G2 are isomorphic in polynomial 
time. 


Prove that in the case that Gi and Ga are isomorphic we can obtain the permutation 7 
mapping G to G2 using the procedure of the above exercise. Use this to complete the proof 
in Example 8.22 and show that graph isomorphism has a checker. Specifically, you have to 
show that if the program claims that Gi = G2 then we can do some further investigation 
(including calling the programs on other inputs) and with high probability conclude that 
either (a) conclude that the program was right on this input or (b) the program is wrong on 
some input and hence is not a correct program for graph isomorphism. 


Define a language L to be downward self reducible there's a polynomial-time algorithm R that 
for any n and x € {0,1}", R’»-1(x) = L(x) where by Lẹ we denote an oracle that solves L 
on inputs of size at most k. Prove that if L is downward self reducible than L € PSPACE. 


Show that MIP C NEXP. 


Show that if we redefine multiprover interactive proofs to allow, instead of two provers, as 
many as m(n) = poly(n) provers on inputs of size n, then the class MIP is unchanged. 
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8.A Interactive proof for the Permanent 


The permanent is defined as follows: 


DEFINITION 8.29 
Let A € F"*” be a matrix over the field F. The permanent of A is: 


perm(A) = > [Leow 


o€Sy i=1 


The problem of calculating the permanent is clearly in PSPACE. In Chapter 9 we will see that if 
the permanent can be computed in polynomial time then P = NP, and hence this problem likely 
does not have a polynomial-time algorithm. 

Although the existence of an interactive proof for the Permanent follows from that for #SAT 
and TQBF, we describe a specialized protocol as well. This is both for historical context (this 
protocol was discovered before the other two protocols) and also because this protocol may be 
helpful for further research. (One example will appear in a later chapter.) 
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We use the following observation: 


X11 X12 sés Tin 
. 21 Nu see LI 

PLL Lo En) := perm . 
Tn,1 Tn,2 sés Inn 


is a degree n polynomial since 


f(£1,£2,..-, En) = WE 


o€Sy i=1 


We now show two properties of the permanent problem. The first is random self reducibility, earlier 
encountered in Section ??: 


THEOREM 8.30 (LIPTON ’88) 
There is a randomized algorithm that, given an oracle that can compute the permanent on 1 — 3 
fraction of the inputs in F"%” (where the finite field F has size > 3n), can compute the permanent 


on all inputs correctly with high probability. 


PROOF: Let A be some input matrix. Pick a random matrix R €g F"*” and let B(x) := A+ z- R 
for a variable x. Notice that: 


e f(x) := perm(B) is a degree n univariate polynomial. 


e For any fixed b 4 0, B(b) is a random matrix, hence the probability that oracle computes 


perm(B(b)) correctly is at least 1 — $. 


Now the algorithm for computing the permanent of A is straightforward: query oracle on all 
matrices {B(i)|1 < 1 < n +1}. According to the union bound, with probability of at least 
1— ztl = 3 the oracle will compute the permanent correctly on all matrices. 

Recall the fact (see Section ?? in Appendix A) that given n + 1 (point, value) pairs [(a;,b;)|i € 
[in +1]), there exists a unique a degree n polynomial p that satisfies Vi p(a;) = bj. Therefore, given 
that the values B(i) are correct, the algorithm can interpolate the polynomial B(x) and compute 
B(0)=4. E 


Note: The above theorem can be strengthened to be based on the assumption that the oracle can 
compute the permanent on a fraction of > +e for any constant e > 0 of the inputs. The observation 
is that not all values of the polynomial must be correct for unique interpolation. See Chapter ?? 


Another property of the permanent problem is downward self reducibility, encountered earlier in 
context of SAT: 


n 
perm(A) = ‘3 ay;perm(Aj i), 
i=l 
where A1; is a (n—1) x (n— 1) sub-matrix of A obtained by removing the 1’st row and i’th column 
of A (recall the analogous formula for the determinant uses alternating signs). 
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DEFINITION 8.31 
Define a (n— 1) x (n— 1) matrix Da(x), such that each entry contains a degree n polynomial. This 
polynomial is uniquely defined by the values of the matrices (41 ¿|i € [n]}. That is: 


Vi € [n] . DAG) = Ari 


Where Da(i) is the matrix D,(x) with i substituted for x. (notice that these equalities force n 
points and values on them for each polynomial at a certain entry of D(x), and hence according 
to the previously mentioned fact determine this polynomial uniquely) 


Observation: perm(Da(x)) is a degree n(n — 1) polynomial in z. 


8.A.1 The protocol 


We now show an interactive proof for the permanent (the decision problem is whether perm(A) = k 
for some value k): 


e Round 1: Prover sends to verifier a polynomial g(x) of degree n(n — 1), which is supposedly 
perm(Da(z)). 


e Round 2: Verifier checks whether: 


k = y a1 g(t) 
i=1 


If not, rejects at once. Otherwise, verifier picks a random element of the field bı ER F and 
asks the prover to prove that g(b¡) = perm(Da(b1)). This reduces the matrix dimension to 
(n — 2) x (n— 2). 


e Round 2(n — 1) — 1: Prover sends to verifier a polynomial of degree 2, which is supposedly 
the permanent of a 2 x 2 matrix. 


e Round 2(n — 1): Verifier is left with a 2 x 2 matrix and calculates the permanent of this 
matrix and decides appropriately. 


CLAIM 8.32 
The above protocol is indeed an interactive proof for perm. 


Proor: If perm(A) = k, then there exists a prover that makes the verifier accept with probability 
1, this prover just returns the correct values of the polynomials according to definition. 

On the other hand, suppose that perm(A) #4 k. If on the first round, the polynomial g(x) sent is 
the correct polynomial D(x), then: 


k 240) aigli) = perm(A) 
i=l 
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And the verifier would reject. Hence g(x) 4 Da(x). According to the fact on polynomials stated 
above, these polynomials can agree on at most n(n — 1) points. Hence, the probability that they 
would agree on the randomly chosen point bı is at most mA. The same considerations apply to 
all subsequent rounds if exist, and the overall probability that the verifier will not accepts is thus 


(assuming |F| > 10n? and sufficiently large n): 


o-a) (=e) (1) 
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Chapter 9 


Complexity of counting 


“It is an empirical fact that for many combinatorial problems the detection of the 
existence of a solution is easy, yet no computationally efficient method is known 
for counting their number.... for a variety of problems this phenomenon can be 
explained.” 

L. Valiant 1979 


The class NP captures the difficulty of finding certificates. However, in many contexts, one is 
interested not just in a single certificate, but actually counting the number of certificates. This 
chapter studies #P, (pronounced “sharp p”), a complexity class that captures this notion. 

Counting problems arise in diverse fields, often in situations having to do with estimations of 
probability. Examples include statistical estimation, statistical physics, network design, and more. 
Counting problems are also studied in a field of mathematics called enumerative combinatorics, 
which tries to obtain closed-form mathematical expressions for counting problems. To give an 
example, in the 19th century Kirchoff showed how to count the number of spanning trees in a graph 
using a simple determinant computation. Results in this chapter will show that for many natural 
counting problems, such efficiently computable expressions are unlikely to exist. 

Here is an example that suggests how counting problems can arise in estimations of probability. 


EXAMPLE 9.1 
In the GraphReliability problem we are given a directed graph on n nodes. Suppose we are told that 
each node can fail with probability 1/2 and want to compute the probability that node 1 has a 
path to n. 

A moment's thought shows that under this simple edge failure model, the remaining graph is 
uniformly chosen at random from all subgraphs of the original graph. Thus the correct answer is 


1 
zn (number of subgraphs in which node 1 has a path to n.) 


We can view this as a counting version of the PATH problem. 


p9.1 (171) 
Complexity Theory: A Modern Approach. ©) 2006 Sanjeev Arora and Boaz Barak. References and attributions are 
still incomplete. 
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In the rest of the chapter, we study the complexity class #P, a class containing the GraphReliability 
problem and many other interesting counting problems. We will show that it has a natural and 
important complete problem, namely the problem of computing the permanent of a given matrix. 
We also show a surprising connection between PH and #P, called Toda’s Theorem. Along the way 
we encounter related complexity classes such as PP and P. 


9.1 The class #P 


We now define the class #P. Note that it contains functions whose output is a natural number, 
and not just 0/1. 


DEFINITION 9.2 (#P) 
A function f : {0,1}" — N is in #P if there exists a polynomial p : N > N and a 
polynomial-time TM M such that for every x € {0,1}": 


a) = Ly € {0, ppal) : M(x,y) = 1} . 


REMARK 9.3 

As in the case of NP, we can also define #P using non-deterministic TMs. That is, #P consists 
of all functions f such that f(x) is equal to the number of paths from the initial configuration to 
an accepting configuration in the configuration graph GM,x of a polynomial-time NDTM M. 


The big open question regarding #P, is whether all problems in this class are efficiently solvable. 
In other words, whether #P = FP. (Recall that FP is the analog of the class P for functions 
with more than one bit of output, that is, FP is the set of functions from {0,1}* to {0,1}* 
computable by a deterministic polynomial-time Turing machine. Thinking of the output as the 
binary representation of an integer we can identify such functions with functions from {0,1}* to N. 
Since computing the number of certificates is at least as hard as finding out whether a certificate 
exists, if #P = FP then NP = P. We do not know whether the other direction also holds: 
whether NP = P implies that #P = FP. We do know that if PSPACE = P then #P = FP, 
since counting the number of certificates can be done in polynomial space. 

Here are two more examples for problems in #P: 


e #SAT is the problem of computing, given a Boolean formula ¢, the number of satisfying 
assignments for q. 


e #CYCLE is the problem of computing, given a directed graph G, the number of simple cycles 
in G. (A simple cycle is one that does not visit any vertex twice.) 


Clearly, if #SAT € FP then SAT € P and so P = NP. Thus presumably #SAT ¢ FP. How 
about #CYCLE? The corresponding decision problem —given a directed graph decide if it has a 
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cycle—can be solved in linear time by breadth-first-search. The next theorem suggests that the 
counting problem may be much harder. 


Figure 9.1: Reducing Ham to #CYCLE: by replacing every edge in G with the above gadget to obtain G”, every 
simple cycle of length £ in G becomes (2™)* simple cycles in G”. 


THEOREM 9.4 
If #CYCLE € FP, then P = NP. 


PROOF: We show that if ¿CYCLE can be computed in polynomial time, then Ham € P, where Ham 
is the NP-complete problem of deciding whether or not a given digraph has a Hamiltonian cycle 
(i.e., a simple cycle that visits all the vertices in the graph). Given a graph G with n vertices, we 
construct a graph G’ such that G has a Hamiltonian cycle iff G’ has at least nr cycles. 

To obtain G’, replace each edge (u,v) in G by the gadget shown in Figure 9.1. The gadget 
has m = nlogn + 1 levels. It is an acyclic digraph, so cycles in G” correspond to cycles in G. 
Furthermore, there are 2 directed paths from u to v in the gadget, so a simple cycle of length £ 
in G yields (27) simple cycles in G”. 

Notice, if G has a Hamiltonian cycle, then G’ has at least (2”)" > n”? cycles. If G has no 
Hamiltonian cycle, then the longest cycle in G has length at most n — 1. The number of cycles is 
bounded above by n”~!. So G’ can have at most (2)"=1 x n™l< n” cycles. W 


9.1.1 The class PP: decision-problem analog for #P. 


Similar to the case of search problems, even when studying counting complexity, we can often 
restrict our attention to decision problems. 'The reason is that there exists a class of decision 
problems PP such that 

PP = P & #P = FP (1) 


Intuitively, PP corresponds to computing the most significant bit of functions in #P. That is, 
L is in PP if there exists a polynomial-time TM M and a polynomial p : N — N such that for 
every x € {0,1}*, 
ve Le |{ye 10,19% : M(x,y) =1)| > $ 2D 
You are asked to prove the non-trivial direction of (1) in Exercise 1. It is instructive to compare 
the class PP, which we believe contains problem requiring exponential time to solve, with the class 
BPP, which although it has a seemingly similar definition, can in fact be solved efficiently using 
probabilistic algorithms (and perhaps even also using deterministic algorithms, see Chapter 16). 
Note that we do not know whether this holds also for the class of decision problems corresponding 
to the least significant bit of #P, namely OP (see Definition 9.13 below). 
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9.2 +P completeness. 


Now we define #P-completeness. Loosely speaking, a function f is #P-complete if it is in #P and 
a polynomial-time algorithm for f implies that #P = FP. To formally define #P-completeness, 
we use the notion of oracle TMs, as defined in Section 3.5. Recall that a TM M has oracle access 
to a language O C {0,1}" if it can make queries of the form “Is q € O?” in one computational 
step. We generalize this to non-Boolean functions by saying that M has oracle access to a function 
f: {0,1}* — {0,1}", if it is given access to the language O = { (x, i) : f(x); = 1}. We use the same 
notation for functions mapping (0,1)* to N, identifying numbers with their binary representation 
as strings. For a function f : {0,1}* > {0,1}*, we define FP? to be the set of functions that are 
computable by polynomial-time TMs that have access to an oracle for f. 


DEFINITION 9.5 
A function f is #P-complete if it is in #P and every g € #P is in FP/ 


If f € FP then FP? = FP. Thus the following is immediate. 


PROPOSITION 9.6 
If f is #P-complete and f € FP then FP = #P. 


Counting versions of many NP-complete languages such as 3SAT,Ham, and CLIQUE naturally 
lead to #P-complete problems. We demonstrate this with #SAT: 


THEOREM 9.7 
#SAT is #P-complete 


PROOF: Consider the Cook-Levin reduction from any L in NP to SAT we saw in Section 2.3. This 
is a polynomial-time computable function f : {0,1}* — {0,1}* such that for every x € {0,1}*, 
x € L & f(x) € SAT. However, the proof that the reduction works actually gave us more 
information than that. It provided a Levin reduction, by which we mean the proof showed a way 
to transform a certificate that x is in L into a certificate (i.e., satisfying assignment) showing that 
f(x) € SAT, and also vice versa (transforming a satisfying assignment for f(x) into a witness that 
zE L). 

In particular, it means that the mapping from the certificates of x to the assignments of f(x) 
was invertible and hence one-to-one. Thus the number of satisfying assignments for f(x) is equal 
to the number of certificates for x. M 


As shown below, there are #P-complete problems for which the corresponding decision problems 
are in fact in P. 
9.2.1 Permanent and Valiant’s Theorem 
Now we study another problem. The permanent of an n x n matrix A is defined as 


perm(A) = > TI] 4.0 (2) 


0ESn i=1 
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where S» denotes the set of all permutations of n elements. Recall that the expression for the 
determinant is similar 


det(A) = y (ye) lee 
i=l 


0ESn 


except for an additional “sign” term.! This similarity does not translate into computational equiv- 
alence: the determinant can be computed in polynomial time, whereas computing the permanent 
seems much harder, as we see below. 

The permanent function can also be interpreted combinatorially. First, suppose the matrix A 
has each entry in {0,1}. It may be viewed as the adjacency matrix of a bipartite graph G(X, Y, E), 
with X = {z1,...,2n}, Y =1[Y1,.-.,Yn) and {zi yj} € E iff Aj; = 1. Then the term [[i Asoc) 
is 1 iff o is a perfect matching (which is a set of n edges such that every node is in exactly one 
edge). Thus if A is a 0.1 matrix then perm(A) is simply the number of perfect matchings in the 
corresponding graph G and in particular computing perm(A) is in #P. If A is a {—1,0,1} matrix, 
then perm(A) = Lo : [Tian Aiea) = 1}| — Lo : [Tier Aiea) = —1}I, so one can make two calls to a 
#SAT oracle to compute perm(A). In fact one can show for general integer matrices that computing 
the permanent is in FP#SAT (see Exercise 2). 

The next theorem came as a surprise to researchers in the 1970s, since it implies that if perm € 
FP then P = NP. Thus, unless P = NP, computing the permanent is much more difficult then 
computing the determinant. 


THEOREM 9.8 (VALIANT’S THEOREM) 
perm for 0,1 matrices is ##4P-complete. 


Before proving Theorem 9.8, we introduce yet another way to look at the permanent. Consider 
matrix A as the the adjacency matrix of a weighted n-node digraph (with possible self loops). Then 
the expression [ [;._, Aj,o(i) is nonzero iff o is a cycle-cover of A (a cycle cover is a subgraph in which 
each node has in-degree and out-degree 1; such a subgraph must be composed of cycles). We define 
the weight of the cycle cover to be the product of the weights of the edges in it. Thus perm(A) is 
equal to the sum of weights of all possible cycle covers. 


EXAMPLE 9.9 

Consider the graph in Figure 9.2. Even without knowing what the subgraph G” is, we show that 
the permanent of the whole graph is 0. For each cycle cover in G” of weight w there are exactly 
two cycle covers for the three nodes, one with weight +w and one with weight —w. Any non-zero 
weight cycle cover of the whole graph is composed of a cycle cover for G’ and one of these two cycle 
covers. Thus the sum of the weights of all cycle covers of G is 0. 


Mt is known that every permutation o € Sn can be represented as a composition of transpositions, where a 
transposition is a permutation that only switches between two elements in [n] and leaves the other elements intact 
(one proof for this statement is the Bubblesort algorithm). If 71,..., 7m is a sequence of transpositions such that their 
composition equals g, then the sign of ø is equal to +1 if m is even and —1 if m is odd. It can be shown that the 
sign is well-defined in the sense that it does not depend on the representation of g as a composition of transpositions. 
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Figure 9.2: The above graph G has cycle cover weight zero regardless of the choice of G”, since for every cycle cover 
of weight w in G”, there exist two covers of weight +w and —w in the graph G. (Unmarked edges have +1 weight; 
we follow this convention through out this chapter.) 


PROOF OF VALIANT’S THEOREM (THEOREM 9.8): We reduce the #P-complete problem #3SAT 
to perm. Given a boolean formula ¢ with n variables and m clauses, first we shall show how to 
construct an integer matrix A’ with negative entries such that perm(A’) = 4” - (#¢). (#¢ stands 
for the number of satisfying assignments of ¢). Later we shall show how to to get a 0-1 matrix A 
from A’ such that knowing perm(A) allows us to compute perm(A’). 

The main idea is that our construction will result in two kinds of cycle covers in the digraph G” 
associated with A’: those that correspond to satisfying assignments (we will make this precise) and 
those that don’t. We will use negative weights to ensure that the contribution of the cycle covers 
that do not correspond to satisfying assignments cancels out. (This is similar reasoning to the one 
used in Example 9.9.) On the other hand, we will show that each satisfying assignment contributes 
47 to perm(A’), and so perm(A’) = 4” . (#¢). 

To construct G’ from ¢, we combine the following three kinds of gadgets shown in Figure 9.3: 


Variable gadget The variable gadget has two possible cycle covers, corresponding to an assign- 
ment of 0 or 1 to that variable. Assigning 1 corresponds to a single cycle taking all the 
external edges (“true-edges”), and assigning 0 correspond to taking all the self-loops and 
taking the “false-edge”. Each external edge of a variable is associated with a clause in which 
the variable appears. 


Clause gadget The clause gadget is such that the only possible cycle covers exclude at least one 
external edge. Also for a given (proper) subset of external edges used there is a unique cycle 
cover (of weight 1). Each external edge is associated with a variable appearing in the clause. 


XOR gadget We also use a graph called the XOR gadget whose purpose is to ensure that for 
— — 
some pair of edges uu’ and vv”, exactly one of these edges is present in any cycle cover that 
counts towards the final sum. 
——> == A a 
Suppose that we replace a pair of edges uu’ and vv’ in some graph G with the XOR gadget as 
described in Figure count:fig:valiantgad to obtain some graph G’. Then, via similar reasoning 
— 
to Example 9.9, every cycle cover of G of weight w that uses exactly one of the edges uu! and 
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Gadget: 


Symbolic description: 


variable gadget: 


A 
©- e 
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9 
U T 
x A A 


external (true) edges - one per clause 


variable gadget 


WAU a 


external edges 


clause gadget: 


1 


$ 1 
external edges - one per variable 


external edges 
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clause gadget 


XOR gadget: 
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The overall construction: 


variable gadget 


for every variable — vuv 


variable gadget 


NA 


clause gadget 
for every clause 


Figure 9.3: The gadgets used in the proof of Valiant's Theorem. 
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=> 
vv’ is mapped to a set of cycle covers in G” whose total weight is 4w (i.e., the set of covers 
that enter the gadget at u and exit at u’ or enter it at v and exit it at v’), while all the other 
cycle covers of G” have total weight 0 (Exercise 3). For this reason, whenever we replace edges 
— 


—o 
uu’ and vv” with a XOR gadget, we can consider in the analysis only cycle covers that use 
exactly one of these edges, as the other covers do not contribute anything to the total sum. 


The XOR gadgets are used to connect the variable gadgets to the corresponding clause gadgets 
so that only cycle covers corresponding to a satisfying assignment will be counted towards the total 
number of cycle covers. Consider a clause, and a variable appearing in it. Each has an external 
edge corresponding to the other, connected by an XOR gadget. If the external edge in the clause is 
not taken then by the analysis of the XOR gadget the external edge in the variable must be taken 
(and hence the variable is true). Since at least one external edge of each clause gadget has to be 
omitted, each cycle cover that is counted towards the sum corresponds to a satisfying assignment. 
Conversely, for each satisfying assignment, there is a a set of cycle covers with total weight 4°” 
(since they passes through the XOR. gadget exactly 3m times). So perm(G’) = 4°"#6¢. 


Reducing to the case 0,1 matrices. Finally we have to reduce finding perm(G’) to finding 
perm(G), where G is an unweighted graph (or equivalently, its adjacency matrix has only 0,1 
entries). We start by reducing to the case that all edges have weights in {+1}. First, note that 
replacing an edge of weight k by k parallel edges of weight 1 does not change the permanent. 
Parallel edges are not allowed, but we can make edges non-parallel by cutting each edge uv in two 
and inserting a new node w with an edge from u to w, w to v and a self loop at w. To get rid 
of the negative weights, note that the permanent of an n vertex graph with edge weights in {+1} 
is a number g in [—n!,+n!] and hence this permanent can be computed from y = x (mod 2+1) 
where m is sufficiently large (e.g., m = n? will do). But to compute y it is enough to compute 
the permanent of the graph where all weight —1 edges are replaced with edges of weight 2”. Such 
edges can be converted to m edges of weight 2 in series, which again can be transformed to parallel 
edges of weight +1 as above. Mi 


9.2.2 Approximate solutions to #P problems 


Since computing exact solutions to ##P-complete problems is presumably difficult, a natural ques- 
tion is whether we can approximate the number of certificates in the sense of the following definition. 


DEFINITION 9.10 
Let f : {0,1}* — N and a < 1. An algorithm A is an a-approximation for f if for every zx, 


af(x) < A(x) < f(u)/a. 


Not all #P problems behave identically with respect to this notion. Approximating certain 
problems within any constant factor œ > 0 is NP-hard (see Exercise 5). For other problems such 
as 0/1 permanent, there is a Fully polynomial randomized approximation scheme (FPRAS), which 
is an algorithm which, for any e, ô, approximates the function within a factor 1+ e (its answer may 
be incorrect with probability ô) in time poly(n,log1/6,log1/e). Such approximation of counting 
problems is sufficient for many applications, in particular those where counting is needed to obtain 
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estimates for the probabilities of certain events (e.g., see our discussion of the graph reliability 
problem). 

The approximation algorithm for the permanent —as well as other similar algorithms for a 
host of #P-complete problems—use the Monte Carlo Markov Chain technique. The result that 
spurred this development is due to Valiant and Vazirani and it shows that under fairly general 
conditions, approximately counting the number of elements in a set ( membership in which is 
testable in polynomial time) is equivalent —in the sense that the problems are interreducible via 
polynomial-time randomized reductions— to the problem of generating a random sample from the 
set. We will not discuss this interesting area any further. 

Interestingly, if P = NP then every #P problem has an FPRAS (and in fact an FPTAS: i.e., 
a deterministic polynomial-time approximation scheme), see Exercise 6. 


9.3 Toda’s Theorem: PH C P***' 


An important question in the 1980s was the relative power of the polynomial-hierarchy PH and 
the class of counting problems #P. Both are natural generalizations of NP, but it seemed that 
their features— alternation and the ability to count certificates, respectively — are not directly 
comparable to each other. Thus it came as big surprise when in 1989 Toda showed: 


THEOREM 9.11 (TODA’S THEOREM [ToD91]) 
PH c P#S^AT, 


That is, we can solve any problem in the polynomial hierarchy given an oracle to a ##P-complete 
problem. 
REMARK 9.12 
Note that we already know, even without Toda’s theorem, that if #P = FP then NP = P 
and so PH = P. However, this does not imply that any problem in PH can be computed in 
polynomial-time using an oracle to #SAT. For example, one implication of Toda’s theorem is that 
a subexponential (i.e., n° time) algorithm for #SAT will imply such an algorithm for any problem 
in PH. Such an implication is not known to hold from a 220 time algorithm for SAT. 


9.3.1 The class 6P and hardness of satisfiability with unique solutions. 


The following complexity class will be used in the proof: 


DEFINITION 9.13 
A language L in the class OP (pronounced “parity P”) iff there exists a polynomial time NTM M 
such that x € L iff the number of accepting paths of M on input zx is odd. 


Thus, @P can be considered as the class of decision problems corresponding to the least sig- 
nificant bit of a #P-problem. As in the proof of Theorem 9.7, the fact that the standard NP- 
completeness reduction is parsimonious implies the following problem @SAT is @P-complete (under 
many-to-one Karp reductions): 
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DEFINITION 9.14 
Define the quantifier @ as follows: for every Boolean formula y on n variables. @,< {0,1}" p(x) is 


true if the number of «’s such that y(x) is true is odd.? The language SAT consists of all the true 
quantified Boolean formula of the form ®,,.<49,1}» P(x) where ¢ is an unquantified Boolean formula 
(not necessarily in CNF form). 


Unlike the class #P, it is not known that a polynomial-time algorithm for $P implies that 
NP = P. However, such an algorithm does imply that NP = RP since NP can be probabilistically 
reduced to GSAT: 


THEOREM 9.15 (VALIANT-VAZIRANI THEOREM) 
There exists a probabilistic polynomial-time algorithm A such that for every n- 
variable Boolean formula ~ 


p E SAT > Pr[A(y) € @SAT] > E 


p ¢ SAT > Pr[A(p) € SSAT] = 0 


To prove Theorem 9.15 we use the following lemma on pairwise independent hash functions: 


LEMMA 9.16 (VALIANT-VAZIRANI LEMMA [?]) 
Let Hn,x be a pairwise independent hash function collection from (0, 1)” to {0, 1}* and S C {0,1}” 
such that 2172 < |S| < 2171. Then, 


Pr | 
hERHn, kb 


{ce S:h(x)=0"}|=1 >} 


PROOF: For every x € S, let p = 27} be the probability that h(x) = 0* when h Er Hn.x. Note 
that for every x 4 a’, Pr[h(x)=0* A h(a") =0*] = p?. Let N be the random variable denoting the 


number of x € S satisfying h(x) = 0". Note that E[N] =|Slp € [4,4]. By the inclusion-exclusion 
principle 
S| 
ia Pa r[h le la r[h(1)=0% A h(a’) =0"] = [Sip ( 9 JP 


and by the union bound we get that Pr[N > 2] < (15) p?. Thus 


E 


Pr[N = 1] = Pr[N > 1] — Pr[N > 2] > sto -2 (15), > |S|p — |S p? > = 


where the last inequality is obtained using the fact that i < |S|p< 3. E 


Note that if we identify true with 1 and 0 with false then Deeto1yn PL) = Veeqo,1}~ P(x) (mod 2). Also note 
that Decto,1y” p(z) = O gom jais Dreta p(@1,..., En). 
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Proof of Theorem 9.15 


We now use Lemma 9.16 to prove Theorem 9.15. Given a formula y on n variables, our probabilistic 
algorithm A chooses k at random from (2,...,n +1) and a random hash function h Er Hy,. It 
then uses the Cook-Levin reduction to compute a formula 7 on variables x € {0,1}",y € {0,1} 
(for m = poly(n)) such that h(x) = 0 if and only if there exists a unique y such that T(x,y) = 1.3 
The output of A if the formula 


y= Bp p(x) Ar(z, y), 
x€{0,1}",ye{0,1}™ 


It is equivalent to the statement 


BD enn =0", 


xe {0,1}” 


If y is unsatisfiable then y is false, since we'll have no x's satisfying the inner formula and 
zero is an even number. If y is satisfiable, we let S be the set of its satisfying assignments. With 
probability 1/n, k satisfies 2*72 < |S| < 2*, conditioned on which, with probability 1/8, there is a 
unique x such that y(x) A h(a) = 0”. Since one happens to be an odd number, this implies that 4 
is true. W 


REMARK 9.17 (HARDNESS OF UNIQUE SATISFIABILITY) 

The proof of Theorem 9.15 implies the following stronger statement: the existence of an algorithm 
to distinguish between an unsatisfiable Boolean formula and a formula with exactly one satisfying 
assignment implies the existence of a probabilistic polynomial-time algorithm for all of NP. Thus, 
the guarantee that a particular search problem has either no solutions or a unique solution does 
not necessarily make the problem easier to solve. 


9.3.2 Step 1: Randomized reduction from PH to 6P 


We now go beyond NP (that is to say, the Valiant-Vazirani theorem) and show that we can actually 
reduce any language in the polynomial hierarchy to GSAT. 


LEMMA 9.18 
Let c € N be some constant. There exists a probabilistic polynomial-time algorithm A such that 
for every y a Quantified Boolean formula with c levels of alternations, 


Y is true => Pr[A(~) € SAT] > 5 
w is false > Pr[A(w) € @SAT] = 


Before proving the Lemma, let us make a few notations and observations: For a Boolean 
formula y on n variables, let 4+((p) denote the number of satisfying assignments of y. We consider 
also formulae y that are partially quantified. That is, in addition to the n variables y takes as input 


3For some implementations of hash functions, such as the one described in Exercise 4, one can construct directly 
(without going through the Cook-Levin reduction) such a formula 7 that does not use the y variables. 
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it may also have other variables that are bound by a V, 4 or @ quantifiers (for example y can be of 
the form p(x1,...,tn) = Vy € {0,1}"7(a1,...,2n,y) where 7 is, say, a 3CNF Boolean formula). 

Given two (possibly partially quantified) formulae y, y on variables x € {0,1}",y € [0,1)” we 
can construct in polynomial-time an n + m variable formula y- Y and a (max{n,m} + 1)-variable 
formula y + y such that #(y- Y) = #(y)#(y) and Hlp + Y) = Hp) + #(w). Indeed, take 
p: plz, y) = p(x) A ply) and p + (z) = ((20 = 0) A pla, ...,2m)) V ((20 = 1) Aya, ...,2m))- 
For a formula y, we use the notation y+ 1 to denote the formula y+ Y where y is some canonical 
formula with a single satisfying assignment. Since the product of numbers is even iff one of the 
numbers is even, and since adding one to a number flips the parity, for every two formulae y, y as 
above 


Bay) a (Avu) + BD - vay) (3) 
x y 


TY 


Ber) > Ply + 1)(a, 2) (4) 
(Boo) (Bv) + P(e +1): +1) + 1)(x, y, 2) (5) 
x y 


T,Y,Z 
PROOF OF LEMMA 9.18: Recall that membership in a PH-language can be reduced to deciding 
the truth of a quantified Boolean formula with a constant number of alternating quantifiers. The 
idea behind the proof is to replace one-by-one each 3/V quantifiers with a $ quantifier. 

Let 4 be a formula with c levels of alternating 4/V quantifiers, possibly with an initial @ 
quantifier. We transform 4 in probabilistic polynomial-time to a formula y” such that y” has only 
c— 1 levels of alternating 4/V quantifiers, an initial $) quantifier, satisfying (1) if y is false then 
so is Y”, and (2) if Y is true then with probability at least 1 — Tint y is true as well. The lemma 
follows by repeating this step c times. 

For ease of notation, we demonstrate the proof for the case that ~ has a single @ quantifier 
and two additional 3/V quantifiers. We can assume without loss of generality that ~ is of the form 


Y = P dret Veo eE, T, w) 5 
z€{0,1}* 


as otherwise we can use the identities V,P(x) = 74,—P(x) and (4) to transform y into this form. 
The proof of Theorem 9.15 provides for every n, a probabilistic algorithm that outputs a for- 
mula 7 on variables x € {0,1}” and y € {0,1} such that for every nonempty set S C {0,1}”, 
PrlOzet0,13" yefo ym T(2,y)] > 1/(8n). Run this algorithm t = 100c£log n times to obtain the for- 
mulae 7,,...,7¿. Then, for every nonempty set S C {0,1}" the probability that there does not 
exist i € |t] such that Oze(0,1)" ye{0,1}” T(x, y) is TRUE is less than 24/(10c). We claim that this 
implies that with probability at least 1 — 1/(10c), the following formula is equivalent to Y: 


BD 42), (6) 
ze{0,1} 
where 


A(z) = Viet P Vive {o,1}® Ti, y) A p(z, Z, w) 
xE{0,1}”,ye{O,1}”™ 
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Indeed, for every z € {0, 1 define S, = E € {0,1}": Wero pre (z, Zz, w)}. Then, 4 is equivalent 
to ®,. cory Sz! is nonempty. But by the union bound, with probability at least 1—1/(10c) it holds 
that for every z such that S, is nonempty, there exists 7, satisfying ®z,,7;(x, y). This means that 
for every such z, 0(2) is true. On the other hand, if S; is empty then certainly 0(2) is false, implying 
that indeed y is equivalent to (6). 

By applying the identity (5), we can transform (6) into an equivalent formula of the desired 
form 


P Vue (z, Y, Z, w) 


2, £, Y, wW 


for some unquantified polynomial-size formula y’. W 


9.3.3 Step 2: Making the reduction deterministic 
To complete the proof of Toda's Theorem (Theorem 9.11), we prove the following lemma: 


LEMMA 9.19 
There is a (deterministic) polynomial-time transformation T that, for every formula y that is an 
input for SSAT, T (4,1) is an unquantified Boolean formula and 


Y € PSAT >#(~) =-1 (mod 2™*1) 
Y Z @SAT >#(y) =0 (mod 27+1) 


PROOF OF THEOREM 9.11 USING LEMMAS 9.18 AND 9.19.: Let L€ PH. We show that we can 
decide whether an input x € L by asking a single question to a #SAT oracle. For every x € {0,1}”, 
Lemmas 9.18 and 9.19 together imply there exists a polynomial-time TM M such that 


reL> Pr [#(M(a,r))=—-1 (mod 2™*)] > 
r€r{0,1}™ 


rE~EL>Vrepforyr#(M(az,r)) =0 (mod grr”) 


where m is the (polynomial in n) number of random bits used by the procedure described in that 
Lemma. Furthermore, even in the case x € L, we are guaranteed that for every r € {0,1}, 
H(M(z,r)) € {0,—1} (mod 27+1), 

Consider the function that maps two strings r, u into the evaluation of the formula M(x,r) on the 
assignment u. Since this function is computable in polynomial-time, the Cook-Levin transformation 
implies that we can obtain in polynomial-time a CNF formula 6, on variables r, u, y such that for 
every r,u, M(x,r) is satisfied by u if and only if there exist a unique y such that 6,(r, u, y) is true. 
Let f(r) be the number of u, y such that 0, (r, u, y) is true, then 


02) = D> Jen), 


re{0,1}”™ 


But if x g L then f,(r) = 0 (mod 27+1) for every r, and hence #(0,) = 0 (mod 27+1). On the 
other hand, if x € L then f,(r) = —1 (mod 2+!) for between 32" and 2” values of r, and is 
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equal to 0 on the other values, and hence #(6,) 4 0 (mod 2™+1). We see that deciding whether 
x E L can be done by computing #(0,). 


PROOF OF LEMMA 9.19: For every pair of formulae y,7 recall that we defined formulas p +7 and 
p- rT satisfying #(p +7) = #(y) + #(7) and #(y- T) = #(y)#(7), and note that these formulae 
are of size at most a constant factor larger than y,7. Consider the formula 47? + 37* (where 7° for 
example is shorthand for 7 - (T - 7)). One can easily check that 


##(T) =-—1 (mod 22) >#(47r° ae 371) =j (mod 2) (7) 
#(T) = 0 (mod 22) >#(47r° dl 31%) == (mod yaa (8) 
Let Yo = Y and iyı = 4? + 3y}. Let Y* = Vrog(m+11- Repeated use of equations (7), (8) 


shows that if #() is odd, then #(~*) = —1 (mod 2™+1) and if #(w) is even, then #(7*) = 0 


(mod 2™+1), Also, the size of ~* is only polynomially larger than size of y. W 


WHAT HAVE WE LEARNED? 


e The class #P consists of functions that count the number of certificates for a 
given instance. If P 4 NP then it is not solvable in polynomial time. 


Counting analogs of many natural NP-complete problems are #P-complete, 
but there are also #P-complete counting problems for which the correspond- 
ing decision problem is in P. One example for this is the problem perm of 
computing the permanent. 


Surprisingly, counting is more powerful than alternating quantifiers: we can 
solve every problem in the polynomial hierarchy using an oracle to a #P- 
complete problem. 


The classes PP and SP contain the decision problems that correspond to 
the most significant and least significant bits (respectively) of a #P function. 
The class PP is as powerful as #P itself, in the sense that if PP = P then 
FP = FP. We do not know if this holds for pP but do know that every 
language in PH randomly reduces to @P. 


9.4 Open Problems 


e What is the exact power of SAT and #S'AT ? 


e What is the average case complexity of n x n permanent modulo small prime, say 3 or 5 ? 
Note that for a prime p > n, random self reducibility of permanent implies that if permanent 
is hard to compute on at least one input then it is hard to compute on 1 — O(p/n) fraction 
of inputs, i.e. hard to compute on average (see Theorem ??). 
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Chapter notes and history 


The definition of #P as well as several interesting examples of #P problems appeared in Valiant’s 
seminal paper [Val79b]. The #P-completeness of the permanent is from his other paper [Val79a]. 
Toda’s Theorem is proved in [Tod91]. The proof given here follows the proof of [KVVY93] (although 
we use formulas where they used circuits.) 

For an introduction to FPRAS’s for computing approximations to many counting problems, 
see the relevant chapter in Vazirani [Vaz01] ( an excellent resource on approximation algorithms in 
general). 


Exercises 


$1 Let f € #P. Show a polynomial-time algorithm to compute f given access to an oracle for 
some language L € PP (see Remark ??). 
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82 Show that computing the permanent for matrices with integer entries is in FpP%SAT. 


$3 Complete the analysis of the XOR gadget in the proof of Theorem 9.8. Let G be any weighted 
graph containing a pair of edges gu and a , and let G” be the graph obtained by replacing 
these edges with the XOR gadget. Prove that every cycle cover of G of weight w that uses 
exactly one of the edges on is mapped to a set of cycle covers in G’ whose total weight is 
4w, and all the other cycle covers of G” have total weight 0. 


§4 Let k < n. Prove that the following family Hn, is a collection of pairwise independent 
functions from {0,1}” to {0,1}*: Identify {0,1} with the field GF(2). For every k x n 
matrix A with entries in GF(2), and k-length vector b € GF(2)”, H,, contains the function 
hay: GF(2)" > GF(2)* defined as follows: h4 p(x) = Ax + b. 


85 Show that if there is a polynomial-time algorithm that approximates #CYCLE within a factor 
1/2, then P = NP. 


$6 Show that if NP = P then for every f € #P and there is a polynomial-time algorithm 
that approximates f within a factor of 1/2. Can you show the same for a factor of 1 — e for 
arbitrarily small constant e > 0? Can you make these algorithms deterministic? 
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Note that we do not know whether P = NP implies that exact computation of functions in 
#P can be done in polynomial time. 
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$7 Show that every for every language in AC? there is a depth 3 circuit of nPoly(logn) size that 


decides it on 1 — 1/poly(n) fraction of inputs and looks as follows: it has a single O gate at 
the top and the other gates are V, A of fanin at most poly(log n). 
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Chapter 10 
Cryptography 


“From times immemorial, humanity has gotten frequent, often cruel, reminders that 
many things are easier to do than to reverse.” 
L. Levin [Lev] 


SOMEWHAT ROUGH STILL 

The importance of cryptography in today’s online world needs no introduction. Here we focus 
on the complexity issues that underlie this field. The traditional task of cryptography was to allow 
two parties to encrypt their messages so that eavesdroppers gain no information about the message. 
(See Figure 10.1.) Various encryption techniques have been invented throughout history with one 
common characteristic: sooner or later they were broken. 


Figure unavailable in pdf file. 


Figure 10.1: People sending messages over a public channel (e.g., the internet) wish to use encryption so that 
eavesdroppers learn “nothing.” 


In the post NP-completeness era, a crucial new idea was presented: the code-breaker should be 
thought of as a resource-bounded computational device. Hence the security of encryption schemes 
ought to be proved by reducing the task of breaking the scheme into the task of solving some 
computationally intractable problem (say requiring exponential time complexity or circuit size), 
thus one could hope to design encryption schemes that are efficient enough to be used in practice, 
but whose breaking will require, say, millions of years of computation time. 

Early researchers tried to base the security of encyption methods upon the (presumed) in- 
tractability of NP-complete problems. This effort has not succeeded to date, seemingly because 
NP-completeness concerns the intractability of problems in the worst-case whereas cryptography 
seems to need problems that are intractable on most instances. After all, when we encrypt email, 
we require that decryption should be difficult for an eavesdropper for all (or almost all) messages, 
not just for a few messages. Thus the concept most useful in this chapter will be average-case 
complexity!. We will see a class of functions called one-way functions that are easy to compute 


1A problem’s average-case and worst-case complexities can differ radically. For instance, 3COL is NP-complete 


p10.1 (187) 
Complexity Theory: A Modern Approach. ©) 2006 Sanjeev Arora and Boaz Barak. References and attributions are 
still incomplete. 
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but hard to invert for most inputs —they are alluded to in Levin’s quote above. Such functions 
exist under a variety of assumptions, including the famous assumption that factoring integers re- 
quires time super-polynomial time in the integer’s bit-length to solve in the average case (e.g., for 
a product of two random primes). 

Furthermore, in the past two decades, cryptographers have taken on tasks above and beyond the 
basic task of encryption—from implementing digital cash to maintaining the privacy of individuals 
in public databases. (We survey some applications in Section 10.4.) Surprisingly, many of these 
tasks can be achieved using the same computational assumptions used for encryption. A crucial 
ingredient in these developments turns out to be an answer to the question: “What is a random 
string and how can we generate one?” The complexity-theoretic answer to this question leads to the 
notion of a pseudorandom generator, which is a central object; see Section 10.2. This notion is very 
useful in itself and is also a template for several other key definitions in cryptography, including 
that of encryption (see Section 10.4). 


Private key versus public key: Solutions to the encryption problem today come in two distinct 
flavors. In private-key cryptography, one assumes that the two (or more) parties participating in 
the protocol share a private “key” —namely, a statistically random string of modest size—that is 
not known to the eavesdropper?. In a public-key encryption system (a concept introduced by Diffie 
and Hellman in 1976 [DH76]) we drop this assumption. Instead, a party P picks a pair of keys: 
an encryption key and decryption key, both chosen at random from some (correlated) distribution. 
The encryption key will be used to encrypt messages to P and is considered public —i.e., published 
and known to everybody including the eavesdropper. The decryption key is kept secret by P 
and is used to decrypt messages. A famous public-key encryption scheme is based upon the RSA 
function of Example 10.4. At the moment we do not know how to base public key encryption on 
the sole assumption that one-way functions exist and current constructions require the assumption 
that there exist one-way functions with some special structure (such as RSA, factoring-based, and 
Lattice-based one way functions). Most topics described in this chapter are traditionally labeled 
private key cryptography. 


10.1 Hard-on-average problems and one-way functions 


A basic cryptographic primitive is a one-way function. Roughly speaking, this is a function f that is 
easy to compute but hard to invert. Notice that if f is not one-to-one, then the inverse f7!(%) may 
not be unique. In such cases “inverting” means that given f(x) the algorithm is able to produce 
some preimage, namely, any element of f~!(f(a))). We say that the function is one-way function 
if inversion is difficult for the “average” (or “many”) x. Now we define this formally; a discussion 
of this definition appears below in Section 10.1.1. A function family (gn) is a family of functions 
where gn takes n-bit inputs. It is polynomial-time computable if there is a polynomial-time TM 
that given an input x computes gj,\(2). 


on general graphs, but on most n-node graphs is solvable in quadratic time or less. A deeper study of average case 
complexity appears in Chapter 15. 

?Practically, this could be ensured with a face-to-face meeting that might occur long before the transmission of 
messages. 
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DEFINITION 10.1 (ONE-WAY FUNCTION) 
A family of functions {fa : {0,1}" > (0,177) is e(n) one-way with security s(n) if it is 
polynomial-time computable and furthermore for every algorithm A that runs in time s(n), 


Preto [4 inverts fn(x)] < e(n). (1) 


Now we give a few examples and discuss the evidence that they are hard to invert “on average 
inputs.” 


EXAMPLE 10.2 
The first example is motivated by the fact that finding the prime factors of a given integer is 
the famous FACTORING problem, for which the best current algorithm has running time about 
20(n2/8) (and even that bounds relies on the truth of some unproven conjectures in number theory). 
The hardest inputs for current algorithms appear to be of the type z - y, where x,y are random 
primes of roughly equal size. 

Here is a first attempt to define a one-way function using this observation. Let {fn} be a family 
of functions where fy: [0,1)” x 40,1)” — (0, 1)” is defined as fn((2]2,[y]2) = [x - y)o. If £ and y 
are primes —-which by the Prime Number Theorem happens with probability 0(1/n?) when x, y 
are random n-bit integers— then fn seems hard to invert. It is widely believed that there are 
c > 1, f >0 such that family fn is (1 — 1/n%)-one-way with security parameter a 

An even harder version of the above function is obtained by using the existence of a randomized 
polynomial-time algorithm A (which we do not describe) that, given 1”, generates a random n-bit 
prime number. Suppose A uses m random bits, where m = poly(n). Then A may be seen as 
a (deterministic) mapping from m-bit strings to n-bit primes. Now let function fm map (71,12) 
to [A(r1) - A(ra)]2, where A(r1), A(r2) are the primes output by A using random strings r1,r2 
respectively. This function seems hard to invert for almost all r,,72. (Note that any inverse (rj, r5) 
for fin(r1,72) allows us to factor the integer A(71) - A(r2) since unique factorization implies that 
the prime pair A(rj), A(r)) must be the same as A(r1), A(r2).) It is widely conjecture that there 
are c > 1, f > 0 such that fn is 1 /n°-one-way with security parameter on 


The FACTORING problem, a mainstay of modern cryptography, is of course the inverse of 
multiplication. Who would have thought that the humble multiplication, taught to children in 
second grade, could be the source of such power? The next two examples also rely on elementary 
mathematical operations such as exponentiation, albeit with modular arithmetic. 


EXAMPLE 10.3 

Let p1, p2,... be a sequence of primes where p; has i bits. Let g; be the generator of the group 
Zp» the set of numbers that are nonzero mod p;. Then for every y € 1,..,p; — 1, there is a unique 
x € {1,..,p — 1} such that 


gi =y (mod pi). 
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Then x — g? (mod p;) is a permutation on 1,..,p; — 1 and is conjectured to be one-way. The 
inversion problem is called the DISCRETE LOG problem. We show below using random self- 
reducibility that if it is hard on worst-case inputs, then it is hard on average. 


We list some more conjectured one-way functions. 


EXAMPLE 10.4 

RSA function. Let m = pq where p, q are large random primes and e be a random number coprime 
to d(m) = (p — 1)(q — 1). Let Zf, be the set of integers in [1,...,m] coprime to m. Then the 
function is defined to be fp,q,e(£) = x° (mod m). This function is used in the famous RSA public- 
key cryptosystem. 

Rabin function. For a composite number m, define f(x) = z? (mod m). If we can invert 
this function on a 1/poly(log m) fraction of inputs then we can factor m in poly(logm) time (see 
exercises). 

Both the RSA and Rabin functions are useful in public-key cryptography. They are examples 
of trapdoor one-way functions: if the factors of m (the “trapdoor” information) are given as well 
then it is easy to invert the above functions. Trapdoor functions are fascinating objects but will 
not be studied further here. 

Random subset sum. Let m = 10n. Let the inputs to f be n positive m-bit integers a1, a2,...,Qn, 
and a subset S of {1,2,...,n}. Its output is (a1, @2,...,4n, » ¡eg ai). Note that f maps n(m-+1)-bit 
inputs to nm + m bits. 

When the inputs are randomly chosen, this function seems hard to invert. It is conjectured that 
there is c > 1,d > 0 such that this function is 1/n°-one-way with security gra. 


10.1.1 Discussion of the definition of one-way function 


We will always assume that the the one-way function under consideration is such that the security 
parameter s(n) is superpolynomial, i.e., larger than n* for every k > 0. The functions described 
earlier are actually believed to be one-way with a larger security parameter 2” for some fixed e > 0. 

Of greater interest is the error parameter e(n), since it determines the fraction of inputs for 
which inversion is easy. Clearly, a continuum of values is possible, but two important cases to 
consider are (i) e(n) = (1 — 1/n°) for some fixed c > 0, in other words, the function is difficult to 
invert on at least 1/n* fraction of inputs. Such a function is often called a weak one-way function. 
The simple one-way function f,, of Example 10.2 is conjectured to be of this type. (ii) e(n) < 1/n* 
for every k > 1. Such a function is called a strong one-way function. 

Yao showed that if weak one-way functions exist then so do strong one-way functions. We will 
prove this surprising theorem (actually, something close to it) in Chapter 17. We will not use 
it in this chapter, except as a justification for our intuition that strong one-way functions exist. 
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(Another justification is of course the empirical observation that the candidate one-way functions 
mentioned above do seem appear difficult to invert on most inputs.) 


10.1.2 Random self-reducibility 


Roughly speaking, a problem is random-self-reducible if solving the problem on any input x reduces 
to solving the problem on a sequence of random inputs y;,ya,..., Where each y; is uniformly 
distributed among all inputs. To put it more intuitively, the worst-case can be reduced to the 
average case. Hence the problem is either easy on all inputs, or hard on most inputs. (In other 
words, we can exclude the possibility that problem is easy on almost all the inputs but not all.) If 
a function is one-way and also randomly self-reducible then it must be a strong one-way function. 
This is best illustrated with an example. 


THEOREM 10.5 

Suppose A is an algorithm with running time t(n) that, given a prime p, a generator g for Z;, 
and an input g*( mod p), manages to find x for 6 fraction of x € Z}. Then there is a randomized 
algorithm A’ with running time O steer (t(n) + poly(n))) that solves DISCRETE LOG on every 
input with probability at least 1 — e. 


PROOF: Suppose we are given y = g*(_ mod p) and we are trying to find x. Repeat the following 
trial O(1/(6 log 1/€)) times: “Randomly pick r € {0,1,...,p—2} and use A to try to compute 
the logarithm of y - g"(modp). Suppose A outputs z. Check if g*~"(modp) is y, and if so, output 
z — r(mod (p — 1)) as the answer.” 

The main observation is that if r is randomly chosen, then y-g"( mod p) is randomly distributed 
in Z% and hence the hypothesis implies that A has a 6 chance of finding its discrete log. After 
O(1/(6 log 1/e) trials, the probability that A failed every time is at most e. W 


COROLLARY 10.6 
If for any infinite sequence of primes p1,p2,..., DISCRETE LOG mod p; is hard on worst-case 
x € Z;,, then it is hard for almost all x. 


Later as part of the proof of Theorem 10.14 we give another example of random self-reducibility: 
linear functions over GF(2). 


10.2 What is a random-enough string? 


Cryptography often becomes much easier if we have an abundant supply of random bits. Here is 
an example. 


EXAMPLE 10.7 (ONE-TIME PAD) 

Suppose the message sender and receiver share a long string r of random bits that is not available 
to eavesdroppers. Then secure communication is easy. To encode message m € {0,1}", take the 
first n bits of r, say the string s. Interpret both strings as vectors in GF(2)” and encrypt m by the 
vector m+ s. The receiver decrypts this message by adding s to it (note that s + s = 0 in GF(2)”). 
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If s is statistically random, then so is m + s. Hence the eavesdropper provably cannot obtain even 
a single bit of information about m regardless of how much computational power he expends. 
Note that reusing s is a strict no-no (hence the name “one-time pad”). If the sender ever 
reuses s to encrypt another message m’ then the eavesdropper can add the two vectors to obtain 
(m+ 8) + (m + s) =m +m, which is some nontrivial information about the two messages. 
Of course, the one-time pad is just a modern version of the old idea of using “codebooks” with 
a new key prescribed for each day. 


One-time pads are conceptually simple, but impractical to use, because the users need to agree 
in advance on a secret pad that is large enough to be used for all their future communications. It 
is also hard to generate because sources of quality random bits (e.g., those based upon quantum 
phenomena) are often too slow. Cryptography’s suggested solution to such problems is to use a 
pseudorandom generator. This is a deterministically computable function g: 0,1)” > {0,1}" (for 
some c > 1) such that if x € {0,1}” is randomly chosen, then g(x) “looks” random. Thus so long as 
users have been provided a common n-bit random string, they can use the generator to produce n° 
“random looking” bits, which can be used to encrypt n°! messages of length n. (In cryptography 
this is called a stream cipher.) 

Clearly, at this point we need an answer to the question posed in the Section’s title! Philosophers 
and statisticians have long struggled with this question. 


EXAMPLE 10.8 

What is a random-enough string? Here is Kolmogorov’s definition: A string of length n is random 
if no Turing machine whose description length is < 0.99n (say) outputs this string when started on 
an empty tape. This definition is the “right” definition in some philosophical and technical sense 
(which we will not get into here) but is not very useful in the complexity setting because checking 
if a string is random according to this definition is undecidable. 

Statisticians have also attempted definitions which boil down to checking if the string has the 
“right number” of patterns that one would expect by the laws of statistics, e.g. the number of times 
11100 appears as a substring. (See Knuth Volume 3 for a comprehensive discussion.) It turns out 
that such definitions are too weak in the cryptographic setting: one can find a distribution that 
passes these statistical tests but still will be completely insecure if used to generate the pad for the 
one-time pad encryption scheme. 


10.2.1 Blum-Micali and Yao definitions 


Now we introduce two complexity-theoretic definitions of pseudorandomness due to Blum-Micali 
and Yao in the early 1980s. For a string y € {0,1}” and S C [n], we let y|s denote the projection 
of Y to the coordinates of S. In particular, y|ņ. denotes the first i bits of y. 
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The Blum-Micali definition is motivated by the observation that one property (in fact, the defin- 
ing property) of a statistically random sequence of bits y is that given y|j,_j, we cannot predict 
Yi+1 with odds better than 50/50 regardless of the computational power available to us. Thus one 
could define a “pseudorandom” string by considering predictors that have limited computational 
resources, and to show that they cannot achieve odds much better than 50/50 in predicting y;+1 
from Yi. Of course, this definition has the shortcoming that any single finite string would be 
predictable for a trivial reason: it could be hardwired into the program of the predictor Turing 
machine. To get around this difficulty the Blum-Micali definition (and also Yao's definition below) 
defines pseudorandomness for distributions of strings rather than for individual strings. Further- 
more, the definition concerns an infinite sequence of distributions, one for each input size. 


DEFINITION 10.9 (BLUM-MICALI) 

Let {gn} be a polynomial-time computable family of functions, where gn : {0,1}" — {0,1}” and 
m = m(n) >n. We say the family is (e(n), t(n))-unpredictable if for every probabilistic polynomial- 
time algorithm A that runs in time t(n) and every large enough input size n, 


1 
Pr[A(g(2) p14) = 9lz)i1] < 5 + e(n), 
where the probability is over the choice of x € {0,1}",7 € {1,...,n}, and the randomness used by 
A. 
If for every fixed k, the family {gn} is (1/n*, n*)-unpredictable for every c > 1, then we say in 
short that it is unpredictable by polynomial-time algorithms. 


REMARK 10.10 
Allowing the tester to be an arbitrary polynomial-time machine makes perfect sense in a crypto- 
graphic setting where we wish to assume nothing about the adversary except an upperbound on 
her computational power. 

Pseudorandom generators proposed in the pre-complexity era, such as the popular linear or 
quadtratic congruential generators do not satisfy the Blum-Micali definition because bit-prediction 
can in fact be done in polynomial time. 


Yao gave an alternative definition in which the tester machine is given access to the entire string 
at once. This definition implicitly sets up a test of randomness analogous to the more famous Turing 
test for intelligence (see Figure 10.2). The tester machine A is given a string y € {0,1}” that is 
produced in one of two ways: it is either drawn from the uniform distribution on (0, pp or 
generated by taking a random string x € {0,1}” and stretching it using a deterministic function 
g: {0,1}" — {0, qe The tester is asked to output “1” if the string looks random to it and 0 
otherwise. We say that g is a pseudorandom generator if no polynomial-time tester machine A has 
a great chance of being able to determine which of the two distributions the string came from. 


DEFINITION 10.11 ([Yao82]) 

Let {gn} be a polynomial-time computable family of functions, where gn: {0,1}" — {0,1}” and 
m = mín) > n. We say it is a (0(n), s(n))-pseudorandom generator if for every probabilistic 
algorithm A running in time s(n) and for all large enough n 


[Pr eton" [A(y) = 1] — Przeto,” [A(g9n(£)) = 1]| < d(n). (2) 
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We call 6(n) the distinguishing probability and s(n) the security parameter. 
If for every c’,k > 1, the family is (1/ ne, n*)-pseudorandom then we say in short that it is a 
pseudorandom generator. 


Figure unavailable in pdf file. 


Figure 10.2: Yao's definition: If c > 1 then g: {0,1}" — {0,1}” is a pseudorandom generator if no polynomial- 
time tester has a good chance of distinguishing between truly random strings of length n° and strings generated by 
applying g on random n-bit strings. 


10.2.2 Equivalence of the two definitions 


Yao showed that the above two definitions are equivalent —up to minor changes in the security 
parameter, a family is a pseudorandom generator iff it is (bitwise) unpredictable. The hybrid 
argument used in this proof has become a central idea of cryptography and complexity theory. 

The nontrivial direction of the equivalence is to show that pseudorandomness of the Blum- 
Micali type implies pseudorandomness of the Yao type. Not surprisingly, this direction is also 
more important in a practical sense. Designing pseudorandom generators seems easier for the 
Blum-Micali definition —as illustrated by the Goldreich-Levin construction below— whereas Yao’s 
definition seems more powerful for applications since it allows the adversary unrestricted access to 
the pseudorandom string. Thus Yao’s theorem provides a bridge between what we can prove and 
what we need. 


THEOREM 10.12 (PREDICTION VS. INDISTINGUISHABILITY [?]) 

Let Let gn:{0,1}" > (0, 1,10) be a family of functions where N(n) = n* for some 
k>l. 

If gn is (> 2t(n))-unpredictable where t(n) > N(n)? then it is (e(n),t(n))- 
pseudorandom. 

Conversely, if gn is (e(n), t(n))-pseudorandom, then it is (e(n), t(n))-unpredictable. 


PROOF: The 


converse part is trivial since a bit-prediction algorithm can in particular be used to distinguish g(x) 
from random strings of the same length. It is left to the reader. 

Let N be shorthand for N(n). Suppose g is not (e(n),t(n))-pseudorandom, and A is a distin- 
guishing algorithm that runs in t(n) time and satisfies: 


[A(g(x)) = 1] — [A(y) =1]| > e(n). (3) 


Pr r 
ZEB ye{0,1}% 
By considering either A or the algorithm that is A with the answer flipped, we can assume that 
the |-| can be removed and in fact 

Pr [A(g(x)) =1]- Pr [A(y) = 1] > e(n). (4) 
2EB ye{0,1}” 
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Consider B, the following bit-prediction algorithm. Let its input be g(x)|<; where x € (0, 1)” 
andi € {0,..., N — 1} are chosen uniformly at random. B’s program is: “Pick bits uj41, Ui+2,..., UN 
randomly and run A on the input g(a) |<;uizititg...un. If A outputs 1, output u;y1 else output 
U;+1.” Clearly, B runs in time less than t(n) + O(N(n)) < 2t(n). To complete the proof we show 
that B predicts g(x);41 correctly with probability at least 5 + ele). 

Consider a sequence of N + 1 distributions Dp through Dy defined as follows (in all cases, 


x € {0,1}” and uy, u2,...,un E {0,1} are assumed to be chosen randomly) 


Do = 4194344 +++ UN 


Dı = g(x)1u2U3 +++ UN 
Di = g(£)<iUi+1 UN 


Dy = g(z)iglx)2 + glx)n 


Furthermore, we denote by D; the distribution obtained from D; by flipping the ith bit (i.e., 
replacing g(a); by g(x);). If D is any of these 2(N +1) distributions then we denote Pryep[A(y) = 1] 
by q(D). With this notation we rewrite (4) as 


a(Dy) — q(Do) > e(n). (5) 
Furthermore, in D;, the (¿+ 1)th bit is equally likely to be g(x);+1 and g(x);+1, so 


a(D;) = 5(a(Diy1) + a(Diz1)), (6) 


Now we analyze the probability that B predicts g(1);+1 correctly. Since i is picked randomly we 
have 


1,1 


— 1 
Pr[B is correct] 23 a (Pri [B’s guess for g(1);+1 is correct | ui+1 = g(£)i+1] 
i=0 

+ P 


r[B's guess for g(1);+1 is correct | ui+1 = Tom) : 
T,U 


Since B’s guess is uj;+1 iff A outputs 1 this is 


y NA 
o (a(Di+1) + 1 — q(Di41)) 
1=0 
r iN 
= 3 toy 2 (Pim) — a(Di41)) 
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From (6), ¢(Di+1) — a(Di+1) = 2(q(Di+1) — a(Di)), so this becomes 


i. 1 
= F 2(q(Di+1) — a(D;)) 
1=0 
= ; + ~(a(Dw) — q(Do)) 
1,0) 
2 N` 


This finishes our proof. M 


10.3 One-way functions and pseudorandom number generators 


Do pseudorandom generators exist? Surprisingly the answer (though we will not prove it in full 
generality) is that they do if and only if one-way functions exist. 


THEOREM 10.13 
One-way functions exist iff pseudorandom generators do. 


Since we had several plausible candidates for one-way functions in Section 10.1, this result helps 
us design pseudorandom generators using those candidate one-way functions. If the pseudorandom 
generators are ever proved to be insecure, then the candidate one-way functions were in fact not 
one-way, and so we would obtain (among other things) efficient algorithms for FACTORING and 
DISCRETE LOG. 

The “if” direction of Theorem 10.13 is trivial: if g is a pseudorandom generator then it must 
also be a one-way function since otherwise the algorithm that inverts g would be able to distinguish 
its outputs from random strings. The “only if” direction is more difficult and involves using a 
one-way function to explicitly construct a pseudorandom generator. We will do this only for the 
special case of one-way functions that are permutations, namely, they map {0,1}” to {0,1}" in a 
one-to-one and onto fashion. As a first step, we describe the Goldreich-Levin theorem, which gives 
an easy way to produce one pseudorandom bit, and then describe how to produce n° pseudorandom 
bits. 


10.3.1 Goldreich-Levin hardcore bit 


Let {fn} be a one-way permutation where fn:{0,1}” — (0, 1)”. Clearly, the function g:{0,1}” x 
{0,1}" — {0,1}?” defined as g(x,r) = (f(x),r) is also a one-way permutation. Goldreich and 
Levin showed that given (f(x),r), it is difficult for a polynomial-time algorithm to predict x © r, 
the scalar product of x and r (mod 2). Thus even though the string (f(x),r) in principle contains 
all the information required to extract (x,r), it is computationally difficult to extract even the 
single bit O r. This bit is called a hardcore bit for the permutation. Prior to the Goldreich-Levin 
result we knew of hardcore bits for some specific (conjectured) one-way permutations, not all. 
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THEOREM 10.14 (GOLDREICH-LEVIN THEOREM) 
Suppose that {fn} is a family of e(n))-one-way permutation with security s(n). Let 


S(n) = (min { s(n), ay ee Then for all algorithms A running in time S(n) 


Proyeto Anla), r) = 2 Or] $5 + Og). (7) 


PROOF: Sup- 


pose that some algorithm A can predict Or with probability 1/2+6 in time t(n). We show how to 
invert f,(x) for O(9) fraction of the inputs in O(n°t(n)/6*) time, from which the theorem follows. 


CLAIM 10.15 
Suppose that 


1 
Pry -efo,1}” [AVia(z), r) =a @7| > 3 +ô. 


Then for at least 6 fraction of x’s 


Prre(0,lA( f(x), 1) = 2 Or] > 


(8) 


(9) 


PROOF: We use an averaging argument. Suppose that p is the fraction of «’s satisfying (9). We 


have p- 1+ (1 — p)(1/2 + 8/2) > 1/2+ ô. Solving this with respect to p, we obtain 


> > 
ETE 


We design an inversion algorithm that given fn(x), where x Er {0,1}”, will try to recover zx. 
It succeeds with high probability if x is such that (9) holds, in other words, for at least ô fraction 
of x. Note that the algorithm can always check the correctness of its answer, since it has f,(1) 


available to it and it can apply fn to its answer and see if f(x) is obtained. 


WARMUP: Reconstruction when the probability in (9) is > 3/4 +0. 


Let P be any program that computes some unknown linear function over GF(2)” but errs on 


some inputs. Specifically, there is an unknown vector x € GF(2)” such that 


Pr[P(r) =g- r] =3/4+ ô. 


(10) 


Then we show to add a simple “correction” procedure to turn P into a probabilistic program 


P’ such that 
Yr PrP ie) err] ol 4. 


(11) 


(Once we know how to compute zx -r for every r with high probability, it is easy to recover x 
bit-by-bit using the observation that if e; is the n-bit vector that is 1 in the ith position and zero 


elsewhere then z - e; = ai, the ith bit of a.) 
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“On input r, repeat the following trial O(logn/0?) times. Pick y randomly from GF(2)” and 
compute the bit P(r+y)+P(y). At the end, output the majority value.” 

The main observation is that when y is randomly picked from GF(2)” then r+ y and y are 
both randomly distributed in GF(2)”, and hence the probability that P(r + y) 4% a- (r+ y) or 
P(y) %a-yis at most 2- (1/4 — ô) = 1/2 — 26. Thus with probability at least 1/2 + 26, each trial 
produces the correct bit. Then Chernoff bounds imply that probability is at least 1 — 1/n? that 
the final majority is correct. 


GENERAL CASE: 

The idea for the general case is very similar, the only difference being that this time we want to 
pick r1,...,7%m so that we already “know” æ © ri. The preceding statement may appear ridiculous, 
since knowing the inner product of x with m > n random vectors is, with high probability, enough 
to reconstruct x (see exercises). The explanation is that the r;’s will not be completely random. 
Instead, they will be pairwise independent. Recall the following construction of a set of pairwise 
independent vectors: Pick k random vectors tı, t2,...,t € GF(2)” and for each nonempty S € 
{1,...,k} define Ys = > ¡egti- This gives 2k — 1 vectors and for S Æ S$’ the random variables 
Ys, Ys are independent of each other. 

Now let us describe the observation at the heart of the proof. Suppose m = 2* — 1 and our 
random strings r1,...,7m are {Ys}’s from the previous paragraph. Then x © Ys = xO (O) ¡eg ti) = 
Jest O ti. Hence if we know z © t; fori = 1,...,k, we also know x O Ys. Of course, we don't 
actually know x © t; for i = 1,...,k since x is unknown and the t;'s are random vectors. But we 
can just try all 2* possibilities for the vector (2 © tj)i=1,...k and run the rest of the algorithm for 
each of them. Whenever our “guess” for these innerproducts is correct, the algorithm succeeds in 
producing x and this answer can be checked by applying fn on it (as already noted). Thus the 
guessing multiplies the running time by a factor 2", which is only m. This is why we can assume 
that we know x © Yg for each subset S. 

The details of the rest of the algorithm are similar to before. Pick m pairwise independent 
vectors Ys’s such that, as described above, we “know” x © Ys for all S. For each i = 1,2,...,n, 
and each S run A on the input (fn(x), Ys O es) (where Ys O e; is Ys with its ith entry flipped). 
Compute the majority value of A(fa(1), Ys ® ei) - 1 O Ys among all S’s and use it as your guess 
for xi. 

Suppose x € GF(2)” satisfies (9). We will show that this algorithm produces all n bits of zx 
with probability at least 1/2. Fix i. For each i, the guess for x; is a majority of m bits. The 
expected number of bits among these that agree with 2; is m(1/2 + 9/2), so for the majority vote 
to result in the incorrect answer it must be the case that the number of incorrect values deviates 
from its expectation by more than md/2. Now, we can bound the variance of this random variable 
and apply Chebyshev’s inequality (Lemma A.16 in Appendix A) to conclude that the probability 
of such a deviation is < 5. 

Here is the calculation using Chebyshev’s inequality. Let Es denote the event that A produces 
the correct answer on (fn(x), Ys ® ex). Since x satisfies (9) and Ys Ge; is randomly distributed over 
GF (2)", we have E(€s) = 1/2+6/2 and Var(€s) = E(€s)(1-E(€s)) < 1. Let € = ¿Es denote the 
number of correct answers on a sample of size m. By linearity of expectation, E[€] = m(1/2+ 6/2). 
Furthermore, the Yg’s are pairwise independent, which implies that the same is true for the outputs 
€s's produced by the algorithm A on them. Hence by pairwise independence Var(€) < m. Now, by 
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Chebyshev’s inequality, the probability that the majority vote is incorrect is at most ae < 5. 


Finally, setting m > 8/nô?, the probability of guessing the ith bit incorrectly is at most 1/2n. 
By the union bound, the probability of guessing the whole word incorrectly is at most 1/2. Hence, 
for every x satisfying (9), we can find the preimage of f(x) with probability at least 1/2, which 
makes the overall probability of inversion at least 6/2. The running time is about m?nx (running 
time of A), which is a x t(n), as we had claimed. W 


10.3.2 Pseudorandom number generation 


We saw that if f is a one-way permutation, then g(x,r) = (f(x),r,x © r) is a pseudorandom 
generator that stretches 2n bits to 2n + 1 bits. Stretching to even more bits is easy too, as we 
now show. Let f'(x) denote the i-th iterate of f on x (i.e., f(f(f(---(f(x))))) where f is applied 


i times). 


THEOREM 10.16 
If f is a one-way permutation then gy(x,r) = (r,2 © r, f(x) Or, f?(£)©r,..., f(x) Or) isa 
pseudorandom generator for N = n° for any constant c > 0. 


PROOF: Since any distinguishing machine could just reverse the string as a first step, it clearly 
suffices to show that the string (r, fY (x) Or, NU) Or,..., f(x) Or, 2 ©r) looks pseudorandom. 
By Yao’s theorem (Theorem 10.12), it suffices to show the difficulty of bit-prediction. For contra- 
diction’s sake, assume there is a PPT machine A such that when x,r € {0,1}" and i € {1,..., N} 
are randomly chosen, 


Pr[A predicts f'(x) © r given (r, f(x) Or, FX +(e) Or,..., f(z) Or) > ; + €. 


We describe an algorithm B that given f(z),r where z,r € (0,1)” are randomly chosen, predicts 
the hardcore bit z © r with reasonable probability, which contradicts Theorem 10.14. 

Algorithm B picks i € {1,..., N} randomly. Let x € {0,1}" be such that f'(1) = z. There is 
of course no efficient way for B to find z, but for any | > 1, B can efficiently compute f**!(x) = 
f'1(f(2))! So it produces the string r, f^ (x) Or, NU) Or,..., fx) Or and uses it as input 
to A. By assumption, A predicts f'(1) © r = z © r with good odds. Thus we have derived a 
contradiction to Theorem 10.14. Mi 


10.4 Applications 


Now we give some applications of the ideas introduced in the chapter. 


10.4.1 Pseudorandom functions 


Pseudorandom functions are a natural generalization of (and are easily constructed using) pseudo- 
random generators. This is a function g: {0,1}’" x {0,1}” — {0,1}'". For each K € {0,1} we 
denote by g|x the function from {0,1}” to {0,1} defined by glx(x) = g(K, x). Thus the family 
contains 2™ functions from {0,1}" to 40,1)”, one for each K. 
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We say g is a pseudorandom function generator if it passes a “Turing test” of randomness 
analogous to that in Yao’s definition of a pseudorandom generator (Definition 10.11). 

Recall that the set of all functions from {0,1}" to {0,1}, denoted Frm , has cardinality 
(2™)?". The PPT machine is presented with an “oracle” for a function from (0,1)” to 40,1)”. The 
function is one of two types: either a function chosen randomly from Fn,m, or a function f|x where 
K € {0,1} is randomly chosen. The PPT machine is allowed to query the oracle in any points 
of its choosing. We say f|x is a pseudorandom function generator if for all c > 1 the PPT has 
probability less than n~© of detecting which of the two cases holds. (A completely formal definition 
would resemble Definition 10.1 and talk about a family of generators, one for each n. Then m is 
some function of n.) 


Figure unavailable in pdf file. 


Figure 10.3: Constructing a pseudorandom function from {0,1}” to {0,1}™ using a random key K € {0,1} and 
a length-doubling pseudorandom generator g: {0,1}”" — (0, 12”. 


Now we describe a construction of a pseudorandom function generator g from a length-doubling 
pseudorandom generator f:f0,1)” — {0, pan, For any K € {0,1}" let Tk be a complete binary 
tree of depth n whose each node is labelled with an m-bit string. The root is labelled K. If a node 
in the tree has label y then its left child is labelled with the first m bits of f(y) and the right child 
is labelled with the last m bits of f(y). Now we define g(K, x). For any x € {0,1}” interpret x as 
a label for a path from root to leaf in Tx in the obvious way and output the label at the leaf. (See 
Figure 10.3.) 

We leave it as an exercise to prove that this construction is correct. 

A pseudorandom function generator is a way to turn a random string K into an implicit de- 
scription of an exponentially larger “random looking” string, namely, the table of all values of the 
function g|x. This has proved a powerful primitive in cryptography; see the next section. Further- 
more, pseudorandom function generators have also figured in a very interesting explanation of why 
current lowerbound techniques have been unable to separate P from NP; see Chapter ??. 


10.4.2 Private-key encryption: definition of security 


We hinted at a technique for private-key encryption in our discussion of a one-time pad (including 
the pseudorandom version) at the start of Section 10.2. But that discussion completely omitted 
what the design goals of the encryption scheme were. This is an important point: design of insecure 
systems often traces to a misunderstanding about the type of security ensured (or not ensured) by 
an underlying protocol. 

The most basic type of security that a private-key encryption should ensure is semantic security. 
Informally speaking, this means that whatever can be computed from the encrypted message is also 
computable without access to the encrypted message and knowing only the length of the message. 
The formal definition is omitted here but it has to emphasize the facts that we are talking about 
an ensemble of encryption functions, one for each message size (as in Definition 10.1) and that the 
encryption and decryption is done by probabilistic algorithms that use a shared private key, and 
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that for every message the guarantee of security holds with high probability with respect to the 
choice of this private key. 

Now we describe an encryption scheme that is semantically secure. Let f:{0,1}" x {0,1}" = 
{0,1}" be a pseudorandom function generator. The two parties share a secret random key K € 
{0,1}". When one of them wishes to send a message x € [0,1)” to the other, she picks a random 
string r € (0,1)” and transmits (r,x € fx(r)). To decrypt the other party computes fx(r) and 
then XORs this string with the last n bits in the received text. 

We leave it as an exercise to show that this scheme is semantically secure. 


10.43 Derandomization 


The existence of pseudorandom generators implies subexponential deterministic algorithms for 
BPP: this is usually referred to as derandomization of BPP. (In this case, the derandomization 
is only partial since it results in a subexponential deterministic algorithm. Stronger complexity 
assumptions imply a full derandomization of BPP, as we will see in Chapter 16.) 


‘THEOREM 10.17 
If for every c > 1 there is a pseudorandom generator that is secure against circuits of size n°, then 
BPP C n.-soDTIME(2” ). 


PROOF: Let us fix an e > 0 and show that BPP C DTIME(2” ). 

Suppose that M is a BPP machine running in n* time. We can build another probabilistic 
machine M’ that takes n€ random bits, streches them to n” bits using the pseudorandom generator 
and then simulates M using this n* bits as a random string. Obviously, M’ can be simulated by 
going over all binary strings nf, running M’ on each of them, and taking the majority vote. 

It remains to prove that M and M’ accept the same language. Suppose otherwise. Then there 
exists an infinite sequence of inputs 71,...,%p,... on which M distinguishes a truly random string 
from a pseudorandom string with a high probability, because for M and M’ to produce different 
results, the probability of acceptance should drop from 2/3 to below 1/2. Hence we can build a 
distinguisher similar to the one described in the previous theorem by hardwiring these inputs into 
a circuit family. M 


The above theorem shows that the existence of hard problems implies that we can reduce the 
randomness requirement of algorithms. This “hardness versus randomness” tradeoff is studied more 
deeply in Chapter 16. 

REMARK 10.18 
There is an interesting connection to discrepancy theory, a field of mathematics. Let S be a set of 
subsets of (0, 1)”. Subset A C {0,1}” has discrepancy e with respect to S if for every s € S, 


IsQ A] [Al 
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Our earlier result that BPP C P/poly showed the existence of polynomial-size sets A that have 
low discrepancy for all sets defined by polynomial-time Turing machines (we only described dis- 
crepancy for the universe {0,1}” but one can define it for all input sizes using lim sup). The goal 
of derandomization is to explicitly construct such sets; see Chapter 16. 
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10.4.4 ‘Tossing coins over the phone and bit commitment 


How can two parties A and B toss a fair random coin over the phone? (Many cryptographic 
protocols require this basic primitive.) If only one of them actually tosses a coin, there is nothing 
to prevent him from lying about the result. The following fix suggests itself: both players toss a 
coin and they take the XOR as the shared coin. Even if B does not trust A to use a fair coin, he 
knows that as long as his bit is random, the XOR is also random. Unfortunately, this idea also 
does not work because the player who reveals his bit first is at a disadvantage: the other player 
could just “adjust” his answer to get the desired final coin toss. 

This problem is addressed by the following scheme, which assumes that A and B are polynomial 
time turing machines that cannot invert one-way permutations. The protocol itself is called bit 
commitment. First, A chooses two strings za and ra of length n and sends a message (f,(1A),7A), 
where fn is a one-way permutation. This way, A commits the string xa without revealing it. Now 
B selects a random bit b and conveys it. Then A reveals x4 and they agree to use the XOR of b 
and (xa Ora) as their coin toss. Note that B can verify that x4 is the same as in the first message 
by applying f,, therefore A cannot change her mind after learning B’s bit. On the other hand, by 
the Goldreich—Levin theorem, B cannot predict x4 O ra from A’s first message, so this scheme is 
secure. 


10.4.5 Secure multiparty computations 


This concerns a vast generalization of the setting in Section 10.4.4. There are k parties and the ith 
party holds a string x; € {0,1}”". They wish to compute f (x1, £2,..., £k) where f :{0,1}"" > {0,1} 
is a polynomial-time computable function known to all of them. (The setting in Section 10.4.4 is 
a subcase whereby each x; is a bit —randomly chosen as it happens—and f is XOR.) Clearly, 
the parties can just exchange their inputs (suitably encrypted if need be so that unauthorized 
eavesdroppers learn nothing) and then each of them can compute f on his/her own. However, this 
leads to all of them knowing each other’s input, which may not be desirable in many situations. 
For instance, we may wish to compute statistics (such as the average) on the combination of several 
medical databases that are held by different hospitals. Strict privacy and nondisclosure laws may 
forbid hospitals from sharing information about individual patients. (The original example Yao 
gave in introducing the problem was of k people who wish to compute the average of their salaries 
without revealing their salaries to each other.) 

We say that a multiparty protocol for computing f is secure if at the end no party learns 
anything new apart from the value of f(x1,22,...,2%). The formal definition is inspired by the 
definition of a pseudorandom generator, and states that for each i, the bits received by party i 
during the protocol should be computationally indistinguishable from completely random bits’. 

It is completely nonobvious why such protocols must exist. Yao [Yao86] proved existence for 
k = 2 and Goldreich, Micali, Wigderson [GMW87] proved existence for general k. We will not 


3Returning to our medical database example, we see that the hospitals can indeed compute statistics on their 
combined databases without revealing any information to each other —at least any information that can be extracted 
feasibly. Nevetheless, it is unclear if current privacy laws allow hospitals to perform such secure multiparty protocols 
using patient data— an example of the law lagging behind scientific progress. 
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describe this protocol in any detail here except to mention that it involves “scrambling” the circuit 
that computes f. 


10.4.6 Lowerbounds for machine learning 


In machine learning the goal is to learn a succinct function f : {0,1}" — {0,1} from a sequence 
of type (a1, f(x1)), (z2, f(x2)),..., where the 2;’s are randomly-chosen inputs. Clearly, this is 
impossible in general since a random function has no succinct description. But suppose f has a 
succinct description, e.g. as a small circuit. Can we learn f in that case? 

The existence of pseudorandom functions implies that even though a function may be polynomial- 
time computable, there is no way to learn it from examples in polynomial time. In fact it is possible 
to extend this impossibility result (though we do not attempt it) to more restricted function families 
such as NC! (see Kearns and Valiant [KV94]). 


10.5 Recent developments 


The earliest cryptosystems were designed using the SUBSET SUM problem. They were all shown to 
be insecure by the early 1980s. In the last few years, interest in such problems —and also the related 
problems of computing approximate solutions to the shortest and nearest lattice vector problems— 
has revived, thanks to a one-way function described in Ajtai [Ajt96], and a public-key cryptosystem 
described in Ajtai and Dwork [AD97] (and improved on since then by other researchers). These 
constructions are secure on most instances iff they are secure on worst-case instances. (The idea 
used is a variant of random self-reducibility. ) 

Also, there has been a lot of exploration of the exact notion of security that one needs for various 
cryptographic tasks. For instance, the notion of semantic security in Section 10.4.2 may seem quite 
strong, but researchers subsequently realized that it leaves open the possibility of some other kinds 
of attacks, including chosen ciphertext attacks, or attacks based upon concurrent execution of 
several copies of the protocol. Achieving security against such exotic attacks calls for many ideas, 
most notably zero knowledge (a brief introduction to this concept appears in Section ??). 


Chapter notes and history 


In the 1940s, Shannon speculated about topics reminiscent of complexity-based cryptography. The 
first concrete proposal was made by Diffie and Hellman [DH76], though their cryptosystem was later 
broken. The invention of the RSA cryptosystem (named after its inventors Ron Rivest, Adi Shamir, 
and Len Adleman) [RSA78] brought enormous attention to this topic. In 1981 Shamir [Sha83] 
suggested the idea of replacing a one-time pad by a pseudorandom string. He also exhibited a weak 
pseudorandom generator assuming the average-case intractability of the RSA function. The more 
famous papers of Blum and Micali [BM84] and then Yao [Yao82] laid the intellectual foundations 
of private-key cryptography. (The hybrid argument used by Yao is a stronger version of one in 
an earlier important manuscript of Goldwasser and Micali [GM84] that proposed probabilistic 
encryption schemes.) The construction of pseudorandom functions in Section 10.4.1 is due to 
Goldreich, Goldwasser, and Micali [G@GM86]. The question about tossing coins over a telephone 


p10.18 (204) 10.5. RECENT DEVELOPMENTS 


was raised in an influential paper of Blum [Blu82]. Today complexity-based cryptography is a vast 
field with several dedicated conferences. Goldreich [Gol04]'s two-volume book gives a definitive 
account. 

A scholarly exposition of number theoretic algorithms (including generating random primes 
and factoring integers) appears in Victor Shoup’s recent book [?] and the book of Bach and Shal- 
lit [BS96]. 

Theorem 10.13 and its very technical proof is in Hastad et al. [HILL99] (the relevant conference 
publications are a decade older). 

Our proof of the Goldreich-Levin theorem is usually attributed to Rackoff (unpublished). 


Exercises 


$1 Show that if P = NP then one-way functions and pseudorandom generators do not exist. 


§2 (Requires just a little number theory). Prove that if some algorithm inverts the Rabin func- 
tion fm(x) = 1? (mod m) on a 1/poly(logm) fraction of inputs then we can factor m in 


poly(log m) time. 
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§3 Show that if f is a one-way permutation then so is f* (namely, f(f(f(---(f(z))))) where f 
is applied k times) where k = n° for some fixed c > 0. 


$4 Assuming one-way functions exist, show that the above fails for one-way functions. 
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$5 Suppose a € GF(2)” is an unknown vector. Let r1,r2,..., "m € GF(2)” be randomly 
chosen, and a © r; revealed to us for all ¿ = 1,2,...,m. Describe a deterministic algorithm 
to reconstruct a from this information, and show that the probability (over the choice of the 
r;’s) is at least 1/4 that it works. 


'OIOZUOU SI JUVUTULIOJOP UTEJIOOD V YY} MOYS 09 pƏU NOA :PUTH 
This shows that the “trick” in Goldreich-Levin’s proof is necessary. 


$6 Suppose somebody holds an unknown n-bit vector a. Whenever you present a randomly 
chosen subset of indices S C {1,...,n}, then with probability at least 1/2 + e, she tells you 
the parity of the all the bits in a indexed by S. Describe a guessing strategy that allows you 
to guess a (an n bit string!) with probability at least (£)° for some constant c > 0. 

87 Suppose g: {0,1}" — {0,1}"*! is any pseudorandom generator. Then use g to describe a 
pseudorandom generator that stretches n bits to n* for any constant k > 1. 


88 Show the correctness of the pseudorandom function generator in Section 10.4.1. 
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$9 Formalize the definition of semantic security and show that the encryption scheme in Sec- 
tion 10.4.2 is semantically secure. 
¿agns st} 
səop YAA ‘SUIT}IIOSTe ouwTy-[eTUIOUATOd Aq əjqeysm3unspur ale 
suorydÁ1oue I} fi ‘x sired oSessout [e 10] yey} MOYS SILA :JUIH 


p10.20 (206) 10.5. RECENT DEVELOPMENTS 


Part II 


Lowerbounds for Concrete 
Computational Models 
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Complexity Theory: A Modern Approach. ©) 2006 Sanjeev Arora and Boaz Barak. References and attributions are 
still incomplete. 


DRAFT 
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In the next few chapters the topic will be concrete complexity, the study of lowerbounds on 
models of computation such as decision trees, communication games, circuits, etc. Algorithms or 
devices considered in this lecture take inputs of a fixed size n, and we study the complexity of these 
devices as a function of n. 
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Chapter 11 


Decision Trees 


A decision tree is a model of computation used to study the number of bits of an input that 
need to be examined in order to compute some function on this input. Consider a function f : 
{0,1}" — {0,1}. A decision tree for f is a tree for which each node is labelled with some x;, and 
has two outgoing edges, labelled 0 and 1. Each tree leaf is labelled with an output value 0 or 1. 
The computation on input £x = 2112... proceeds at each node by inspecting the input bit xi 
indicated by the node’s label. If x; = 1 the computation continues in the subtree reached by taking 
the 1-edge. The 0-edge is taken if the bit is 0. Thus input x follows a path through the tree. The 
output value at the leaf is f(x). An example of a simple decision tree for the majority function is 
given in Figure 11.1 


Figure unavailable in pdf file. 


Figure 11.1: A decision tree for computing the majority function Maj(r1, 22,73) on three bits. Outputs 1 if at 
least two input bits are 1, else outputs 0. 


Recall the use of decision trees in the proof of the lower bound for comparison-based sorting 
algorithms. That study can be recast in the above framework by thinking of the input —which 
consisted of n numbers — as consisting of (5) bits, each giving the outcome of a pairwise comparison 
between two numbers. 

We can now define two useful decision tree metrics. 

DEFINITION 11.1 
The cost of tree t on input x, cost(t, x), is the number of bits of x examined by t. 


DEFINITION 11.2 
The decision tree complexity of function f, D(f), is defined as follows, where T below refers to 
the set of decision trees that decide f. 
D = min max cost(t, x 1 
(f) = min se (t, x) (1) 
The decision tree complexity of a function is the number of bits examined by the most efficient 
decision tree on the worst case input to that tree. We are now ready to consider several examples. 
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still incomplete. 
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EXAMPLE 11.3 

(Graph connectivity) Given a graph G as input, in adjacency matrix form, we would like to know 
how many bits of the adjacency matrix a decision tree algorithm might have to inspect in order to 
determine whether G is connected. We have the following result. 

THEOREM 11.4 

Let f be a function that computes the connectivity of input graphs with m vertices. Then D(f) = 
(2)- 

The idea of the proof of this theorem is to imagine an adversary that constructs a graph, edge by 
edge, in response to the queries of a decision tree. For every decision tree that decides connectivity, 
the strategy implicitly produces an input graph which requires the decision tree to inspect each of 
the (3) possible edges in a graph of m vertices. 


Adversary Strategy: 
Whenever the decision tree algorithm asks about edge ei, 
answer “no” unless this would force the graph to be disconnected. 


After i queries, let N; be the set of edges for which the adversary has replied “no”, Y; the set 
of edges for which the adversary has replied “yes”. and E; the set of edges not yet queried. The 
adversary’s strategy maintains the invariant that Y, is a disconnected forest for i < (3) and Y; U E; 
is connected. This ensures that the decision tree will not know whether the graph is connected 
until it queries every edge. 


EXAMPLE 11.5 

(OR Function) Let f(x1,£2,... £n) = Vj_, xi Here we can use an adversary argument to show 
that D(f) = n. For any decision tree query of an input bit x;, the adversary responds that 2; 
equals 0 for the first n — 1 queries. Since f is the OR function, the decision tree will be in suspense 
until the value of the nth bit is revealed. Thus D(f) is n. 


EXAMPLE 11.6 
Consider the AND-OR function, with n = 2". We define fp as follows. 


fk-1(£1, -< Zok-1—1) A fr_1(Xon-1,---Zor) if k is even 
fal21,.-+,%n) = 4 fe1(E1,.. Lon-13) V fk-1(£ok-1,--- £o) if k > 1 and is odd (2) 
Ti ifk=1 
A diagram of a circuit that computes the AND-OR function is shown in Figure 11.2. It is left as 
an exercise to prove, using induction, that D( fp) = 2". 
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Figure 11.2: A circuit showing the computation of the AND-OR function. The circuit has k layers of alternating 
gates, where n = 2". 


11.1 Certificate Complexity 


We now introduce the notion of certificate complexity, which, in a manner analogous to decision 
tree complexity above, tells us the minimum amount of information needed to be convinced of the 
value of a function f on input zx. 


DEFINITION 11.7 

Consider a function f:(0,1)” — {0,1}. If f(x) = 0, then a O-certificate for x is a sequence of 
bits in x that proves f(x) = 0. If f(x) = 1, then a 1-certificate is a sequence of bits in x that 
proves f(x) = 1. 


DEFINITION 11.8 
The certificate complexity C(f) of f is defined as follows. 


C(f) = max {number of bits in the smallest 0- or 1- certificate for x} (3) 


x:input 


EXAMPLE 11.9 

If f is a function that decides connectivity of a graph, a 0-certificate for an input must prove that 
some cut in the graph has no edges, hence it has to contain all the possible edges of a cut of the 
graph. When these edges do not exist, the graph is disconnected. Similarly, a 1-certificate is the 
edges of a spanning tree. Thus for those inputs that represent a connected graph, the minimum 
size of a 1-certificate is the number of edges in a spanning tree, n — 1. For those that represent a 
disconnected graph, a O certificate is the set of edges in a cut. The size of a 0-certificate is at most 
(n/2)? = n?/4, and there are graphs (such as the graph consisting of two disjoint cliques of size 
n/2) in which no smaller 0-certificate exists. Thus C(f) = n?/4. 


EXAMPLE 11.10 

We show that the certificate complexity of the AND-OR function f, of Example 11.6 is 2!*/21. 
Recall that f is defined using a circuit of k layers. Each layer contains only OR-gates or only 
AND-gates, and the layers have alternative gate types. The bottom layer receives the bits of input 
x as input and the single top layer gate outputs the answer f(x). If f(x) = 1, we can construct 
a 1-certificate as follows. For every AND-gate in the tree of gates we have to prove that both its 
children evaluate to 1, whereas for every OR-gate we only need to prove that some child evaluates 
to 1. Thus the 1-certificate is a subtree in which the AND-gates have two children but the OR gates 
only have one each. Thus the subtree only needs to involve 2!*/?! input bits. If f(x) = 0, a similar 
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argument applies, but the role of OR-gates and AND-gates, and values 1 and O are reversed. The 
result is that the certificate complexity of fi is 214/21, or about Jn. 


The following is a rough way to think about these concepts in analogy to Turing machine 
complexity as we have studied it. 


“~~ 
A 
Mejit 


low decision tree complexity => P 


— 
gl 
— 


low 1-certificate complexity ~ NP 


—~ 
D 
Ticani. 


low 0-certificate complexity ~ coNP 


The following result shows, however, that the analogy may not be exact since in the decision tree 
world, P = NP N coNP. It should be noted that the result is tight, for example for the AND-OR 
function. 


THEOREM 11.11 
For function f, D(f) < C(f)?. 


PROOF: Let So, Sı be the set of minimal 0-certificates and 1-certificates, respectively, for f. Let 
k = C(f), so each certificate has at most k bits. 


REMARK 11.12 

Note that every 0O-certificate must share a bit position with every 1-certificate, and furthermore, 
assign this bit differently. If this were not the case, then it would be possible for both a 0-certificate 
and 1-certificate to be asserted at the same time, which is impossible. 


The following decision tree algorithm then determines the value of f in at most k? queries. 

Algorithm: Repeat until the value of f is determined: Choose a remaining 0-certificate from So 
and query all the bits in it. If the bits are the values that prove the f to be 0, then stop. Otherwise, 
we can prune the set of remaining certificates as follows. Since all 1-certificates must intersect the 
chosen 0-certificate, for any cı € S1, one bit in cı must have been queried here. Eliminate cı from 
consideration if the certifying value of cı at at location is different from the actual value found. 
Otherwise, we only need to consider the remaining k — 1 bits of c1. 

This algorithm can repeat at most k times. For each iteration, the unfixed lengths of the 
uneliminated 1-certificates decreases by one. This is because once some values of the input have 
been fixed due to queries, for any 0-certificate, it remains true that all 1-certificates must intersect 
it in at least one location that has not been fixed, otherwise it would be possible for both a 0- 
certificate and a 1-certificate to be asserted. With at most k queries for at most k iterations, a 
total of k? queries is used. W 
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11.2 Randomized Decision Trees 


There are two equivalent ways to look at randomized decision trees. We can consider decision trees 
in which the branch taken at each node is determined by the query value and by a random coin 
flip. We can also consider probability distributions over deterministic decision trees. The analysis 
that follows uses the latter model. 

We will call P a probability distribution over a set of decision trees 7 that compute a particular 
function. P(t) is then the probability that tree t is chosen from the distribution. For a particular 
input x, then, we define c(P,x) = > pnr P(t)cost(t,x). c(P,x) is thus the expected number of 
queries a tree chosen from 7 will make on input x. We can then characterize how well randomized 
decision trees can operate on a particular problem. 


DEFINITION 11.13 
The randomized decision tree complexity, R(f), of f, is defined as follows. 


R(f)= min max c(P, 2) (7) 


The randomized decision tree complexity thus expresses how well the best possible probability 
distribution of trees will do against the worst possible input for a particular probability distribution 
of trees. We can observe immediately that R(f) > C(f). This is because C(f) is a minimum value 
of cost(t,x). Since R(f) is just an expected value for a particular probability distribution of these 
cost values, the minimum such value can be no greater than the expected value. 


EXAMPLE 11.14 

Consider the majority function, f = Maj(x1, 22,23). It is straightforward to see that D(f) = 3. 
We show that R(f) < 8/3. Let P be a uniform distribution over the (six) ways of ordering the 
queries of the three input bits. Now if all three bits are the same, then regardless of the order 
chosen, the decision tree will produce the correct answer after two queries. For such z, c(P, x) = 2. 
If two of the bits are the same and the third is different, then there is a 1/3 probability that the 
chosen decision tree will choose the two similar bits to query first, and thus a 1/3 probability that 
the cost will be 2. There thus remains a 2/3 probability that all three bits will need to be inspected. 
For such zx, then, c(P, x) = 8/3. Therefore, R(f) is at most 8/3. 


How can we prove lowerbounds on randomized complexity? For this we need another concept. 


11.3 Lowerbounds on Randomized Complexity 


NEEDS CLEANUP NOW 

To prove lowerbounds on randomized complexity, it suffices by Yao’s Lemma (see Section 11.6) 
to prove lowerbounds on distributional complexity. Where randomized complexity explores distribu- 
tions over the space of decision trees for a problem, distributional complexity considers probability 
distributions on inputs. It is under such considerations that we can speak of “average case analysis.” 
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Let D be a probability distribution over the space of input strings of length n. Then, if A is 
a deterministic algorithm, such as a decision tree, for a function, then we define the distributional 
complexity of A on a function f with inputs distributed according to D as the expected cost for 
algorithm A to compute f, where the expectation is over the distribution of inputs. 


DEFINITION 11.15 
The distributional complexity d(A,D) of algorithm A given inputs distributed according to D 
is defined as: 

d(A,D) = X` D(«)cost(A, x) = Exepicost(A, x)| (8) 


x:input 


From this we can characterize distributional complexity as a function of a single function f 
itself. 


DEFINITION 11.16 
The distributional decision tree complexity, A( f) of function f is defined as: 


A(f)= max min d(A, D) (9) 
Where A above runs over the set of decision trees that are deciders for f. 


So the distributional decision tree complexity measures the expected efficiency of the most 
efficient decision tree algorithm works given the worst case distribution of inputs. 
The following theorem follows from Yao’s lemma. 


THEOREM 11.17 


RA = A(f). 


So in order to find a lower bound on some randomized algorithm, it suffices to find a lower 
bound on A(f). Such a lower bound can be found by postulating an input distribution D and 
seeing whether every algorithm has expected cost at least equal to the desired lower bound. 


EXAMPLE 11.18 

We return to considering the majority function, and we seek to find a lower bound on A(f). 
Consider a distribution over inputs such that inputs in which all three bits match, namely 000 
and 111, occur with probability 0. All other inputs occur with probability 1/6. For any decision 
tree, that is, for any order in which the three bits are examined, there is exactly a 1/3 probability 
that the first two bits examined will be the same value, and thus there is a 1/3 probability that 
the cost is 2. There is then a 2/3 probability that the cost is 3. Thus the overall expected cost 
for this distribution is 8/3. This implies that A(f) > 8/3 and in turn that R(f) > 8/3. So 


A(f) = R(f) = 8/3. 
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11.4 Some techniques for decision tree lowerbounds 


DEFINITION 11.19 (SENSITIVITY) 
If f:{0,1}" — {0,1} is a function and x € {0,1}” then the sensitivity of f on x, denoted s,(f), 
is the number of bit positions i such that f(x) 4 f(x"), where zt is x with its ith bit flipped. The 
sensitivity of f, denoted s(f), is max, {s,(f)}. 

The block sensitivity of f on x, denoted bsz(f), is the maximum number b such that there are 
disjoint blocks of bit positions B;,2,..., Bẹ such that f(x) 4 f(1P+) where 2% is x with all its bits 
flipped in block B;. The block sensitivity of f denoted bs( f) is max, {bsz(f)}. 


It is conjectured that there is a constant c (as low as 2) such that bs( f) = O(s(f)°) for all f but 
this is wide open. The following easy observation is left as an exercise. 


LEMMA 11.20 
For any function, s(f) < bs(f) < D(f). 


THEOREM 11.21 (NISAN) 


C(f) < s(fybs(f). 


PROOF: For any input x € {0,1}” we describe a certificate for x of size s(f)bs(f). This certificate 
is obtained by considering the largest number of disjoint blocks of variables B1, Bo,..., By, that 
achieve b = bsz(f) < bs(f). We claim that setting these variables according to x constitutes a 
certificate for x. 

Suppose not, and let z’ be an input that is consistent with the above certificate. Let B»,1 be 
a block of variables such that z’ = x4+1, Then By, must be disjoint from B1, B2,... Be, which 
contradicts b = bs,(f). 

Note that each of B1, Bo,..., By has size at most s(f) by definition of s( f), and hence the size 
of the certificate we have exhibited is at most s(f)bs(f). MM 


Recent work on decision tree lowerbounds has used polynomial representations of boolean func- 
tions. Recall that a multilinear polynomial is a polynomial whose degree in each variable is 1. 


DEFINITION 11.22 
An n-variate polynomial p(x1,22,...,tn) represents f : {0,1}" — {0,1} if p(x) = f(x) for all 
x € {0,1}”. 

The degree of f, denoted deg(f), is the degree of the multilinear polynomial that represents f. 


(The exercises ask you to show that the multilinear polynomial representation is unique, so deg( f) 
is well-defined.) 


EXAMPLE 11.23 
The AND of n variables x1, %2,..., Uy is represented by the multilinear polynomial [[/_, z; and 
OR is represented by 1 — []j_,(1 — z). 
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The degree of AND and OR is n, and so is their decision tree complexity. There is a similar 
connection for other problems too, but it is not as tight. The first part of the next theorem is an 
easy exercise; the second part is nontrivial. 


THEOREM 11.24 
1. deg(f) < D(f). 


2. (Nisan-Smolensky) D(f) < deg(f)*bs(f) < O(deg(f)*). 


11.5 Comparison trees and sorting lowerbounds 


TO BE WRITTEN 


11.6 Yao’s MinMax Lemma 


This section presents Yao’s minmax lemma, which is used in a variety of settings to prove lower- 
bounds on randomized algorithms. Therefore we present it in a very general setting. 

Let Y be a finite set of inputs and A be a finite set of algorithms that solve some computational 
problem on these inputs. For x € Y,a € A, we denote by cost(A, x) the cost incurred by algorithm 
A on input x. A randomized algorithm is a probability distribution R on A. The cost of R on 
input x, denoted cost(R, x), is Eser[cost( A, x)|]. The randomized complexity of the problem is 


min max cost(R, x). (10) 
R EX 


Let D be a distribution on inputs. For any deterministic algorithm A, the cost incurred by it 
on D, denoted cost(A,D), is Exep[cost(A, x)|]. The distributional complexity of the problem is 


max min cost(A, D). (11) 
D ACA 


Yao's Lemma says that these two quantitities are the same. It is easily derived from von Neu- 
mann's minmax theorem for zero-sum games, or with a little more work, from linear programming 
duality. 

Yao's lemma is typically used to lowerbound randomized complexity. To do so, one defines 
(using some insight and some luck) a suitable distribution D on the inputs. Then one proves that 
every deterministic algorithm incurs high cost, say C, on this distribution. By Yao’s Lemma, it 
follows that the randomized complexity then is at least C. 


Exercises 


$1 Suppose f is any function that depends on all its bits; in other words, for each bit position 2 
there is an input x such that f(x) 4 f(z’). Show that s(f) = Q(logn). 


$2 Consider an f defined as follows. The n-bit input is partitioned into | yn] blocks of size 
about yn. The function is 1 iff there is at least one block in which two consecutive bits are 1 
and the remaining bits in the block are 0. Estimate s(f),bs(f),C(f), D(f) for this function. 
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$3 Show that there is a unique multilinear polynomial that represents f:(0,1)” — {0,1}. Use 
this fact to find the multilinear representation of the PARITY of n variables. 


§4 Show that deg(f) < D(f). 


Chapter notes and history 


The result that the decision tree complexity of connectivity and many other problems is (5) has 
motivated the following conjecture (atributed variously to Anderaa, Karp, Yao): 

Every monotone graph property has D(-) = (3). 

Here “monotone” means that adding edges to the graph cannot make it go from having the 
property to not having the property (e.g., connectivity). “Graph property” means that the property 
does not depend upon the vertex indices (e.g., the property that vertex 1 and vertex 2 have an 
edge between them). This conjecture is known to be true up to a O(1) factor; the proof uses 
topology and is excellently described in Du and Ko [DK00]. A more ambitious conjecture is that 
even the randomized decision tree complexity of monotone graph properties is Q(n?) but here the 
best lowerbound is close to n*/*. 

The polynomial method for decision tree lowerbounds is surveyed in Buhrman and de Wolf [BdW02]. 
The method can be used to lowerbound randomized decision tree complexity (and more recently, 
quantum decision tree complexity) but then one needs to consider polynomials that approximately 
represent the function. 
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Chapter 12 


Communication Complexity 


Communication complexity concerns the following scenario. There are two players with unlimited 
computational power, each of whom holds an n bit input, say x and y. Neither knows the other’s 
input, and they wish to collaboratively compute f(x,y) where function f:{0,1}” x {0,1}" — (0,1) 
is known to both. Furthermore, they had foreseen this situation (e.g., one of the parties could be 
a spacecraft and the other could be the base station on earth), so they had already —before they 
knew their inputs x, y— agreed upon a protocol for communication!. The cost of this protocol is 
the number of bits communicated by the players for the worst-case choice of x, y. 

Researchers have studied many modifications of the above basic scenario, including randomized 
protocols, nondeterministic protocols, average-case protocols (where x, y are assumed to come from 
a distribution), multiparty protocols, etc. Truly, this is a self-contained mini-world within com- 
plexity theory. Furthermore, lowerbounds on communication complexity have uses in a variety of 
areas, including lowerbounds for parallel and VLSI computation, circuit lowerbounds, polyhedral 
theory, data structure lowerbounds, etc. We give a very rudimentary introduction to this area; an 
excellent and detailed treatment can be found in the book by Kushilevitz and Nisan [KN97]. 


12.1 Definition 


Now we formalize the informal description of communication complexity given above. 
A t-round communication protocol for f is a sequence of function pairs (51, C1), (S2, C2),..., (St, Cz), (Fis fa). 

The input of S; is the communication pattern of the first ¿— 1 rounds and the output is from (1,2), 
indicating which player will communicate in the ¿th round. The input of C; is the input string of 
this selected player as well as the communication pattern of the first i — 1 rounds. The output of 
C; is the bit that this player will communicate in the ith round. Finally, fı, fo are 0/1-valued func- 
tions that the players apply at the end of the protocol to their inputs as well as the communication 
pattern in the t rounds in order to compute the output. These two outputs must be f(x,y). The 


'Do not confuse this situation with information theory, where an algorithm is given messages that have to be 
transmitted over a noisy channel, and the goal is to transmit them robustly while minimizing the amount of com- 
munication. In communication complexity the channel is not noisy and the players determine what messages to 
send. 
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Complexity Theory: A Modern Approach. ©) 2006 Sanjeev Arora and Boaz Barak. References and attributions are 
still incomplete. 
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communication complexity of f is 


C(f)= min max {Number of bits exchanged by P on z, y.) 
protocols P x,y 


Notice, C(f) < n+ 1 since the trivial protocol is for one player to communicate his entire input, 
whereupon the second player computes f(x,y) and communicates that single bit to the first. Can 
they manage with less communication? 


EXAMPLE 12.1 (PARITY) 

Suppose the function f(x,y) is the parity of all the bits in x, y. We claim that C(f) = 2. Clearly, 
C(f) > 2 since the function depends nontrivially on each input, so each player must transmit at 
least one bit. Next, C(f) < 2 since it suffices for each player to transmit the parity of all the bits 
in his possession; then both know the parity of all the bits. 


REMARK 12.2 

Sometimes students ask whether a player can communicate by not saying anything? (After all, 
they have three options: send a 0, or 1, or not say anything in that round.) We can regard such 
protocols as communicating with a ternary, not binary, alphabet, and analyze them analogously. 


12.2 Lowerbound methods 


Now we discuss methods for proving lowerbounds on communication complexity. As a running 
example in this chapter, we will use the equality function: 


1 ifx=y 
EQ(z,y) = 
Ue, y) i otherwise 
We will see that C(EQ) > n. 


12.2.1 Fooling set 


We show C(EQ) > n. For contradiction’s sake, suppose a protocol exists whose complexity is 
at most n — 1. Then there are only 27! communication patterns possible between the players. 
Consider the set of all 2” pairs (x, x). Using the pigeonhole principle we conclude there exist two 
pairs (1,1) and (2’,z’) on which the communication pattern is the same. Of course, thus far we 
have nothing to object to, since the answers EQ(z, x) and EQ(2’, x’) on both pairs are 1. However, 
now imagine giving one player x and the other player 2’ as inputs. A moment's thought shows 
that the communication pattern will be the same as the one on (x, x) and (x”, x’). (Formally, this 
can be shown by induction. If player 1 communicates a bit in the first round, then clearly this bit 
is the same whether his input is x or x’. If player 2 communicates in the 2nd round, then his bit 
must also be the same on both inputs since he receives the same bit from player 1. And so on.) 
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Hence the player's answer on (x, x) must agree with their answer on (x, x’). But then the protocol 
must be incorrect, since EQ(z, 2’) = 0 4 EQ(z, x). 
The lowerbound argument above is called a fooling set argument. It is formalized as follows. 


DEFINITION 12.3 
A fooling set for f:{0,1}" x (0,1)” — {0,1} is a set S C {0,1}” x (0,1)” and a value b € {0,1} 
such that: 


1. For every (x,y) € S, f(x,y) = b. 
2. For every two distinct pairs (1, y1), (2, y2) € S, either f(x1, y2) Æ b or f(x2, y1) Æ b. 


LEMMA 12.4 
If f has a fooling set with m pairs then C(f) > logs m. 


EXAMPLE 12.5 (DISJOINTNESS) 

Let x,y be interpreted as characteristic vectors of subsets of {1,2,...,n}. Let DISJ(z,y) = 1 if 
these two subsets are disjoint, otherwise DISJ(x,y) = 0. Then C(DISJ) > n since the following 2” 
pairs constitute a fooling set: 


S = {(A4, A) : AC {1,2,...,n}}. 


12.2.2 The tiling lowerbound 


The tiling lowerbound takes a more global view of f. Consider the matrix of f, denoted M(f), 
which is a 2” x 2” matrix whose (x,y)'th entry is f(x,y). See Figure 12.1. We visualize the 


Figure unavailable in pdf file. 


Figure 12.1: Matrix M(f) for the equality function when the inputs to the players have 3 bits. The numbers in the 
matrix are values of f. 


communication protocol in terms of this matrix. A combinatorial rectangle (or just rectangle) in 
the matrix is a submatrix corresponding to A x B where A C ([0,1)”, B C {0,1}”. If the protocol 
begins with the first player sending a bit, then M(f) partitions into two rectangles of the type 
Ao x {0,1}", Ay x B”, where A, is the subset of strings for which the first player communicates 
bit b. Notice, Ag U Ai = {0,1}”. If the next bit is sent by the second player, then each of the two 
rectangles above is further partitioned into two smaller rectangles depending upon what this bit 
was. If the protocol continues for k steps, the matrix gets partitioned into 2% rectangles. Note that 
each rectangle in the partition corresponds to a subset of input pairs for which the communication 
sequence thus far has been identical. (See Figure 12.2 for an example.) 
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Figure 12.2: Two-way communication matrix after two steps. The large number labels are the concatenation of 
the bit sent by the first player with the bit sent by the second player. 


If the protocol stops, then the value of f is determined within each rectangle, and thus must be 
the same for all pairs x, y in that rectangle. Thus the set of all communication patterns must lead 
to a partition of the matrix into monochromatic rectangles. (A rectangle A x B is monochromatic 
if for all x in A and y in B, f(x,y) is the same.) 

DEFINITION 12.6 
A monochromatic tiling of M(f) is a partition of M(f) into disjoint monochromatic rectangles. 
We denote by x(f) the minimum number of rectangles in any monochromatic tiling of M(f). 


The following theorem is immediate from our discussion above. 
‘THEOREM 12.7 
If f has communication complexity C then it has a monochromatic tiling with at most 2© rectangles. 
Consequently, C > logs x(f). 


The following observation shows that the tiling bound subsumes the fooling set bound. 
LEMMA 12.8 
If f has a fooling set with m pairs, then x(f) > m. 


PROOF: If (#1, y1) and (x2, y2) are two of the pairs in the fooling set, then they cannot be in a 
monochromatic rectangle since not all of (x1, y1), (v2, ya), (11, Y2), (12, y1) have the same f value. 
A 


12.23 Rank lowerbound 


Now we introduce an algebraic method to lowerbound x(f) (and hence communication complexity). 
Recall the high school notion of rank of a square matrix: it is the size of the largest subset of 
rows/colums that are independent. The following is another definition. 
DEFINITION 12.9 
If a matrix has entries from a field F then the rank of an n x n matrix M is the minimum value 
of l such that M can be expressed as 
l 
M = 5 a; B;, 
i=l 


where a; € FA {0} and each B; is an n x n matrix of rank 1. 


Note that 0,1 are elements of every field, so we can compute the rank over any field we like. The 
choice of field can be crucial; see Problem 5 in the exercises. 

The following theorem is trivial, since each monochromatic rectangle can be viewed (by filling 
out entries outside the rectangle with 0’s) as a matrix of rank at most 1 . 
THEOREM 12.10 
For every function f, x(f) > rank(M(f)). 
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12.2.4 Discrepancy 
The discrepancy of a rectangle A x B in M(f) is 


37m {number of 1’s in Ax B — number of 0’s in A x B|]. (1) 
The discrepancy of the matrix M(f), denote Disc(f), is the largest discrepancy among all 
rectangles. The following Lemma relates it to x(f). 


LEMMA 12.11 


1 


x(f) 2 Disc f) 


Proor: For a monochromatic rectangle, the discrepancy is its size divided by 2°”. The total 
number of entries in the matrix is 2?”. The bound follows. W 


EXAMPLE 12.12 

Lemma 12.11 can be very loose. For the EQ() function, the discrepancy is at least 1—27” (namely, 
the discrepancy of the entire matrix), which would only give a lowerbound of 2 for x(f). However, 
x(f) is at least 2”, as already noted. 


Now we describe a method to upperbound the discrepancy using eigenvalues. 


LEMMA 12.13 (EIGENVALUE BOUND) 
For any matrix M, the discrepancy of a rectangle A x B is at most Amaz(M),/|A||B|/2?", where 
Amax(M) is the magnitude of the largest eigenvalue of M. 


PROOF: Let 14,18 € R” denote the characteristic vectors of A,B. Then |l4|, = Vier P = 


VAI. 


The discrepancy of the rectangle A x B is 
Das 1 ES 1 = 


explain this. 


EXAMPLE 12.14 

The mod 2 inner product function defined as f(x,y) = (1 - y)2 = >, viyi(mod2) has been encoun- 
tered a few times in this book. To bound its discrepancy, we consider the matrix 2M(f) — 1. This 
transformation makes the range of the function {—1,1} and will be useful again later. Let this new 
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matrix be denoted N. It is easily checked that every two distinct rows (columns) of N are orthog- 
onal, every row has l norm 2”/2, and that NT = N. Thus we conclude that N? = 27] where J is 
the unit matrix. Hence every eigenvalue is either +27/2 or —2"/?, and thus Lemma 12.13 implies 
that the discrepancy of a rectangle A x B is at most 2”/2,/|A]]B] and the overall discrepancy is 
at most 2%7/2 (since |A|, |B| < 2”). 


A technique for upperbounding the discrepancy 


Now we describe an upperbound technique for the discrepancy that will later be useful in the 
multiparty setting (Section 12.3). For ease of notation, in this section we change the range of f to 
{—1,1} by replacing 1's in M(f) with —1’s and replacing 0’s with 1’s. Note that now 


1 
Disc(f) = max an y» f(a,d) 
ac A bEB 


DEFINITION 12.15 


E(f) = Es, ,a2,b1,b2 ie []j=1,2 fas, bj)| ; 
Note that E(f) can be computed, like the rank, in polynomial time given the M(f) as input. 
LEMMA 12.16 

Disel f) < Ef)". 


PROOF: The proof follows in two steps. 


CLAM 1: For every function h:{0,1}" x {0,1}" > {1,-1}, E(h) > (Ea plf (a, b)])*. 
We will use the Cauchy-Schwartz inequality, specifically, the version according to which E[z?] > 
(E[2])? for every random variable z. 


E(h) = Eaa |Ev pa | [| [[ Kib) (2) 


i=1,2 j=1,2 
= Eas,az (EolA(an, b)A(az, 0)))”] (3) 
> (Eaa [Eo[h(a1, b)h(az D)? (Cauchy Schwartz) (4) 
> (E, p[h(a, b)})*. beste prev. two steps) (5) 


CLAIM 2: For every function f there is a function h such that E(f) = E(h) and E, p[h(a, b)] > 
Disc(f). 
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First, we note that for every two functions g1, g2:{0,1}" — {—1, 1}, if we define h = fo g1 o ga 
as 


h(a, b) = f(a, b)gı(a)g2(b) 
then E(f) = E(h). The reason is that for all a1, az, b1, ba, 
TL TÍ Kaidi) = g(a)’ (a2)? 92(b1)?92(b2)? J] II Fib) 


i=1,2 j=1,1 i=1,2 j=1,2 


and the square of any value of g1, ga is 1. 


Now we prove Claim 2 using the probabilistic method. Define two random functions 91,92: 
{0,1}" — {-1, 1} as follows: 


Ta Ta € {—1,1} is randomly chosen 


1 ifbe€ B 
g2(b) = 
sp sp € {—1,1} is randomly chosen 


‘ ifac A 


Let h = f o gı o go, and therefore E(h) = E(f). Furthermore 


Ey, g [Eos (h(a,0)]] = Eno [Eos ool f(a,)91(a) 92(0)] (6) 
=>. Y] fas) (7) 

ac A, bEB 
= Dise() (8) 


where the second line follows from the fact that Eg, [g1(a)] = Ey, [g2(0)] = 0 fora ¢ A and b¢ B. 
Thus in particular there exist g1, g2 such that [E, ¿[h(a, b)]| > Disc(f). m 


12.2.5 Comparison of the lowerbound methods 


As already noted, discrepancy upperbounds imply lowerbounds on x(f). Of the other three meth- 
ods, the tiling argument is the strongest, since it subsumes the other two. The rank method is the 
weakest, since the rank lowerbound always implies a tiling lowerbound and a fooling set lowerbound 
(the latter follows from Problem 3 in the exercises). 

Also, we can separate the power of these lowerbound arguments. For instance, we know functions 
for which there is a significant gap between log x(f) and log rank(M(f)). However, the following 
conjecture (we only state one form of it) says that all three methods (except discrepancy, which as 
already noted can be arbitrarily far from x(f)) give the same bound up to a polynomial factor. 


CONJECTURE 12.17 (LOG RANK CONJECTURE) 
There is a constant c > 1 such that C(f) = O(log(rank(M(f)))°) for all f and all input sizes n. 
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12.3 Multiparty communication complexity 


There is more than one way to generalize communication complexity to a multiplayer setting. The 
most interesting model is the “number on the forehead” model often encountered in math puzzles 
that involve people in a room, each person having a bit on their head which everybody else can 
see but they cannot. More formally, there is some function f :({0,1}")* — {0,1}, and the input 
is (11,%2,..., Uy) where each x; € {0,1}". The ith player can see all the x; such that j 4 i. As 
in the 2-player case, the k players have an agreed-upon protocol for communication, and all this 
communication is posted on a “public blackboard”. At the end of the protocol all parties must 
know Pi... £k). 


EXAMPLE 12.18 
Consider computing the function 


n 
f (a1, 22,23) = QD maj(z1i, 22i, 231) 

i=l 
in the 3-party model where 21, 22,x3 are n bit strings. The communication complexity of this 
function is 3: each player counts the number of 2's such that she can determine the majority of 
XL1j4,2;,%3; by examining the bits available to her. She writes the parity of this number on the 
blackboard, and the final answer is the parity of the players’ bits. This protocol is correct because 
the majority for each row is known by either 1 or 3 players, and both are odd numbers. 


EXAMPLE 12.19 (GENERALIZED INNER PRODUCT) 
The generalized inner product function GIP, n maps nk bits to 1 bit as follows 


nmo k 
Foireann) =@ AN (9) 


i=1 j=1 


Notice, for k = 2 this reduces to the mod 2 inner product of Example 12.14. 


In the 2-party model we introduced the notion of a monochromatic rectangle in order to prove 
lower bounds. For the k-party case we will use cylinder intersections. A cylinder in dimen- 
sion i is a subset S of the inputs such that if (x1,...,£ķ) € S then for all x we have that 
(£1, -< , 241,24, 2i41,---, Tk) € S also. A cylinder intersection is NF_,T; where T; is a cylinder in 
dimension ¿. 

As noted in the 2-party case, a communication protocol can be viewed as a way of partitioning 
the matrix M(f). Here M(f) is a k-dimensional cube, and player ¿'s communication does not 
depend upon z;. Thus we conclude that if f has a multiparty protocol that communicates c bits, 
then its matrix has a tiling using at most 2° monochromatic cylinder intersections. 
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LEMMA 12.20 
If every partition of M(f) into monochromatic cylinder intersections requires at least R cylinder 
intersections, then the k-party communication complexity isat least logs R. 


Discrepancy-based lowerbound 


In this section, we will assume as in our earlier discussion of discrepancy that the range of the 
function f is {—1,1}. We define the k-party discrepancy of f by analogy to the 2-party case 


Disc(f) = 7 max y f(a, a2,...,ak)|, 
(a1,a2,.-;ak)ET 
where T ranges over all cylinder intersections. 
To upperbound the discrepancy we introduce the k-party analogue of E(f). Let a cube be a 
set D in {0, 1 of 2" points of the form {41,1, 42,1} X {a1,2, 49,2} X ++: x [a] 4, 09), where each 
Qij € {0, 1%”. 


E(f) = Ep 


II i) 


acD 


Notice that the definition of €() for the 2-party case is recovered when k = 2. The next lemma 
is also an easy generalization. 


LEMMA 12.21 


Dise(f) < (E(f))/". 


PROOF: The proof is analogous to Lemma 12.16 and left as an exercise. The only difference is that 
instead of defining 2 random functions we need to define k random functions g1, 92, gx: (0, i oli > 
{—1,1}, where g; depends on every one of the k coordinates except the ith. Mi 


Now we can prove a lowerbound for the Generalized Inner Product function. Note that since 
we changed the range to {—1, 1} it is now defined as 


GIPrn(t1, £25. - , Ep) = (—1) sn Iye 2i (moa?) , (10) 


THEOREM 12.22 
The function GI Pk n has k-party communication complexity Q(n/ 8") as n grows larger. 


PROOF: We use induction on k. For k > 1 let 6, be defined using 61 = 0 and (6x41 = LEA, We 
claim that 
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Assuming truth for k — 1 we prove for k. A random cube D in {0, po is picked by picking 
a11, 421 € [0,1)” and then picking a random cube D' in (0, 17D”, 


E(GI Pkn) = Eanan | Ep: II GIP. n(@) (11) 


@E {a11,a21}x D’ 


The proof proceeds by considering the number of coordinates where strings a11 and ag, are identical. 
Examining the expression for GI Py in (10) we see that these coordinates contribute nothing once 
we multiply all the terms in the cube, since their contributions get squared and thus become 1. 
The coordinates that contribute are 

TO BE COMPLETED M 


12.4 Probabilistic Communication Complexity 


Will define the model, give the protocol for EQ, and describe the discrepancy-based lowerbound. 


12.5 Overview of other communication models 
We outline some of the alternative settings in which communication complexity has been studied. 


Nondeterministic protocols: These are defined by analogy to NP. In a nondeterministic pro- 
tocol, the players are both provided an additional third input z (“nondeterministic guess” ). 
Apart from this guess, the protocol is deterministic. The cost incurred on 2, y is 


min {|z| + number of bits exchanged by protocol when guess is z}. 
2 


The nondeterministic communication complexity of f is the minimum k such that there is a 
nondeterministic protocol whose cost for all input pairs is at most k. 


In general, one can consider communication protocols analogous to NP, coNP, PH etc. 


Randomized protocols: These are defined by analogy to RP,BPP. The players are provided 
with an additional input r that is chosen uniformly at random from m-bit strings for some 
m. Randomization can significantly reduce the need for communication. For instance we 
can use fingerprinting with random primes (explored in Chapter 7), to compute the equality 
function by exchanging O(log n) bits: the players just pick a random prime p of O(log n) bits 
and exchange x (mod p) and y (mod p). 


Average case protocols: Just as we can study average-case complexity in the Turing machine 
model, we can study communication complexity when the inputs are chosen from a distribu- 
tion D. This is defined as 


Cp(f)= min ) Pejy) € D| x {Number of bits exchanged by P on z, y.) 
x,y 


protocols P 
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Computing a non boolean function: Here the function’s output is not just {0,1} but an m-bit 
number for some m. We discuss one example in the exercises. 


Asymmetric communication: The “cost” of communication is asymmetric: there is some B 
such that it costs the first player B times as much to transmit a bit than it does the second 
player. The goal is to minimize the total cost. 


Multiparty settings: The most obvious generalization to multiparty settings is whereby f has k 
arguments 21, 22,..., £k and player i gets xi. At the end all players must know f(11,12,..., Tk). 
This is not as interesting as the so-called “number of the forehead” where player 7 can see all 
of the input except for xi. We discuss it in Section ?? together with some applications. 


Computing a relation: There is a relation R C {0,1}" x {0,1}” x {1, 2,...,m} and given x, y € 
B” the players seek to agree on any b € {1,2,...,m} such that (x, y,b) € R. See section ??. 


These and many other settings are discussed in [KN97]. 


12.6 Applications of communication complexity 


We briefly discussed parallel computation in Chapter 6. Yao [Yao79] invented communication com- 
plexity as a way to lowerbound the running time of parallel computers for certain tasks. The idea is 
that the input is distributed among many processors, and if we partition these processors into two 
halves, we may lowerbound the computation time by considering the amount of communication 
that must necessarily happen between the two halves. A similar idea is used to prove time/space 
lowerbounds for VLSI circuits. For instance, in a VLSI chip that is an m x m grid, if the communi- 
cation complexity for a function is greater than c, then the time required to compute it is at least 
c/m. 

Communication complexity is also useful in time-space lowerbounds for Turing machines (see 
Problem 1 in exercises), and circuit lowerbounds (see Chapter 13). 

Data structures such as heaps, sorted arrays, lists etc. are basic objects in algorithm design. 
Often, algorithm designers wish to determine if the data structure they have designed is the best 
possible. Communication complexity lowerbounds can be used to establish such results. See [KN97]. 

Yannakakis [Yan91] has shown how to use communication complexity lowerbounds to prove 
lowerbounds on the size of polytopes representing NP-complete problems. Solving the open prob- 
lem mentioned in Problem 8 in the exercises would prove a lowerbound for the polytope representing 
vertex cover. 


Exercises 


g1 If S(n) < n, show that a space S(n) TM takes at least Q(n/S(n)) steps to decide the language 
{xr#zx : x € {0,1}*}. 


§2 Show that the high school definition of rank (the size of the largest set of independent rows 
or columns) is equivalent to that in Definition 12.9. 
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$11 


$12 


Give a fooling set argument that proves that C(f) > [log rank(M(f))]. 


Show that C(f)rank(M(f) +1. 
Consider x,y as vectors over GF(2)” and let f(x,y) be their inner product mod 2. Prove 
that the communication complexity is n. 
"XIBUL J-J 94) 
st f ory f — (f) WZ XLIYeu oy} Jo URI oy} PUNOGISMOT :JUTH 
What field should you use to compute the rank? Does it matter? 


Let f : {0,1}" x {0,1}" — {0,1} be such that all rows of M(f) are distinct. Show that 
C(f) > logn. 


"yueI oY} PUNOGISMO] YUH 
(Aho, Ullman, Yannakakis) Show that C(f) = O(log? x(f)). 


"payeorunururos 393 syq ((£)X30])0 oseyd yore ul 
pue ‘seseyd ((£)X30])0 sey [ooojoid ay y, ur sor, ared-yndut 1oy} 
sopsueqoor |(f)X] ayy Jo Yor aururoyop 09 AI} sioAe[d JL :JUTH 


For any graph G with n vertices, consider the following communication problem: Player 1 
receives a clique C in G, and Player 2 receives an independent set I. They have to com- 
municate in order to determine |C N I|. (Note that this number is either 0 or 1.) Prove an 
O(log? n) upperbound on the communication complexity. 


Can you improve your upperbound or prove a lower bound better than Q(logn)? (Open 
question) 


Prove Lemma 12.21 using the hint given there. 


(Karchmer-Wigderson) Consider the following problem about computing a relation. Associate 
the following communication problem with any function f:(0,1)” — {0,1}. Player 1 gets 
any input x such that f(x) = 0 and player 2 gets any input y such that f(y) = 1. They have 
to communicate in order to determine a bit position į such that x; Æ yi. 


Show that the communication complexity of this problem is exactly the minixmum depth of 


any circuit that computes f. (The maximum fanin of each gate is 2.) 


Use the previous question to show that computing the parity of n bits requires depth at least 
2logn. 


Show that the following computational problem is in EXP: given the matrix M(f) of a 
boolean function, and a number K, decide if C(f) < K. 


(Open since Yao [Yao79]) Can you show this problem is complete for some complexity class? 
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Chapter notes and history 


Communication complexity was first defined by Yao [Yao79]. Other early papers that founded the 
field were Papadimitriou and Sipser [PS84], Mehlhorn and Schmidt [MS82] (who introduced the 
rank lowerbound) and Aho, Ullman and Yannakakis [AUY883]. 

The original log rank conjecture was that C(f) = O(rank(M(f))) but this was disproved by 
Raz and Spieker [RS95]. 

The book by Nisan and Kushilevitz [KN97] is highly recommended. 
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Chapter 13 


Circuit lowerbounds 


Complexity theory’s Waterloo 


We believe that NP does not have polynomial-sized circuits. We’ve seen that if true, this 
implies that NP % P. In the 1970s and 1980s, many researchers came to believe that the route 
to resolving P versus NP should go via circuit lowerbounds, since circuits seem easier to reason 
about than Turing machines. The success in this endeavor was mixed. 

Progress on general circuits has been almost nonexistent: a lowerbound of n is trivial for any 
function that depends on all its input bits. We are unable to prove even a superlinear circuit 
lowerbound for any NP problem— the best we can do after years of effort is 4.5n — o(n). 

To make life (comparatively) easier, researchers focussed on restricted circuit classes, and were 
successful in proving some decent lowerbounds. We prove some of the major results of this area and 
indicate where researchers are currently stuck. In Chapter 22 we’ll explain some of the inherent 
obstacles that need to be overcome to make further progress. 


13.1 AC?’ and Hástad's Switching Lemma 


As we saw in Chapter 6, AC? is the class of languages computable by circuit families of constant 
depth, polynomial size, and whose gates have unbounded fanin. (Constant depth circuits with 
fanin 2 can only compute functions depending on a constant number of input bits.) The burning 
question in the late 1970s was whether problems like Clique and TSP have AC? circuits. However, 
in 1981, Furst, Saxe and Sipser and independently, Ajtai, proved a lowerbound for a much simpler 
function: 


THEOREM 13.1 ([?, ?]) 
Let @ be the parity function. That is, for every x € {0,1}", @(z1,..., £n) = jy, zi (mod 2). 
Then Q g ACC. 


Often courses in digital logic design teach students how to do “circuit minimization” using 
Karnaugh maps. Note that circuits talked about in those courses are depth 2 circuits, i.e. CNF or 
DNF. Indeed, it is easy to show (using for example the Karnaugh map technique studied in logic 


p13.1 (235) 
Complexity Theory: A Modern Approach. ©) 2006 Sanjeev Arora and Boaz Barak. References and attributions are 
still incomplete. 
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design) that the parity function requires exponentially many gates if the depth is two. However, 
those simple ideas do not seem to generalize to even depth 3 circuits. 

The main tool in the proof of Theorem 13.1 is the concept of random restrictions. Let f be a 
function computable by a depth d circuit and suppose that we choose at random a vast majority 
(i.e., n — nf for some constant e > 0 depending on d) of the input variables and assign to each such 
variable either 0 or 1 at random. We'll prove that with positive probability, the function f subject 
to this restriction is constant (i.e., either always zero or always one). Since the parity function 
cannot be made a constant by fixing values to a subset of the variables, it follows that it cannot be 
computed by a constant depth circuit. 


13.1.1 The switching lemma 


Now we prove the main lemma about how a circuit simplifies under a random restriction. A k-DNF 
(resp. k-CNF) formula is an OR of AND’s (resp. AND or OR’s) where each AND (resp. OR) 


involves at most k variables. 


LEMMA 13.2 (HASTAD’S SWITCHING LEMMA [HAs86]) 
Suppose f is expressible as a k-DNF, and let p denote a random restriction that assigns random 
values to t randomly selected input bits. Then for every s > 2. 


_ 4)p.10\ 8/2 
Pr,[fl, is not expressible as s-CNF | < (==) (1) 


n 


where f|, denotes the function f restricted to the partial assignment p. 


We’ll typically use this lemma with k, s constant and t ~ n — y/n in which case the guaranteed 
bound on the probability will be n~° for some constant c. Note that by applying the lemma to the 
function =f, we can get the same result with the terms DNF and CNF interchanged. 


Proving Theorem 13.1 from Lemma 13.2. Now we show how Hastad’s lemma implies that 
parity is not in AC?. We start with any AC? circuit and assume that the circuit has been simplified 
as follows (the simplifications are straightforward to do and are left as Exercises 1 and 2): (a) All 
fanouts are 1; the circuit is a tree (b) All not gates to the input level of the circuit; equivalently, 
the circuit has 2n input wires, with the last n of them being the negations of the first n (c) V and 
A gates alternate —at worst this assumption doubles the depth of the circuit (d) The bottom level 
has A gates of fanin 1. 

We randomly restrict more and more variables, where each step with high probability will reduce 
the depth of the circuit by 1 and will keep the bottom level at a constant fanin. Specifically, letting 
n; stand for the number of unrestricted variables after step 1, we restrict n; — y/n; variables at step 
i+ 1. Since ny = n, we have n; = n1/2. Let n? denote an upper bound on the number of gates in 
the circuit and let k; = 10b2*. We'll show that with high probability, after the i*” restriction we're 
left with a depth-d — i circuit with at most k* fanin in the bottom level. Indeed, suppose that the 
bottom level contains A gates and the level above it contains V gates. The function each such V 


plo y ki+1/2 . 
) , which 


—— 
n1/2+1 


gate computes is a k;-DNF and hence by Lemma 13.2, with probability 1 — ( 
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is at least 1 — 1/(10n?) for large enough n, the function such a gate computes will be expressible 
as a kj41-CNF. We can then merge this CNF with the A-gate above it, reducing the depth of the 
circuit by one (see Figures 13.1 and 13.2). The symmetric reasoning applies in the case the bottom 
level consists of V gates— in this case we use the lemma to transform the k;-CNF of the level 
above it into a kj,;-DNF. Note that we apply the lemma at most once per each of the at most n? 
gates of the original circuit. By the union bound, with probability 9/10, if we continue this process 
for d — 2 steps, we'll get a depth two circuit with fanin k = kg_2 at bottom level (i.e., a k-CNF 
or k-DNF formula). If we then choose to restrict each variable with probability half (i.e., restrict 
about half of the variables to a random value), this circuit will be reduced to a constant function 
with probability at least 27*. Since the parity function is not constant under any restriction of less 
than n variables, this proves Theorem 13.1. E 


Figure unavailable in pdf file. 


Figure 13.1: Circuit before Hastad switching transformation. 
Figure unavailable in pdf file. 


Figure 13.2: Circuit after Hástad switching transformation. Notice that the new layer of A gates can be collapsed 
with the single A parent gate, to reduce the number of levels by one. 


13.1.2 Proof of the switching lemma (Lemma 13.2) 


Now we prove the Switching Lemma. The original proof was more complicated; this one is due 
to Razborov. Let f be expressible as a k-DNF on n variables. Let t be as in the lemma and let 
R+ denote the set of all restrictions to t variables (note we can assume t > n/2). We have that 
[Ral = (1)2!. Let Kis denote the set of restrictions p such that f|, is not a s-CNF. We need to 
bound |K; s|/|R+| by the right hand side of (1) to prove the lemma. We'll do that by showing a 
one-to-one function mapping Kts into the set Z x S where Z is the set of restrictions of at least t+ s 
variables (i.e. Z = Uy>145Ry) and S is some set of size 32s. This will prove the lemma since at he 


=p 
range t' > n/2, (7) Pe ( z y and hence Z will be of size bounded by roughly n2* ES Rel. 


n—t! n 
We leave verifying the exact bound as Exercise 3. 


Mapping K; s into Zx S. Let p € Kis be a restriction fixing t variables such that f|, is not an 
s-CNF. We need to map p in a one-to-one way into some restriction p* of at least t + s variables, 
and some additional element in a set S of size at most 32%, 


Special case: each term has at most one “live” variable. To get some intuition for the 
proof, consider first the case that for each term t in the k-DNF formula for f, p either fixed t to 
the value 0 or left a single unassigned variable in t, in which case we say that t's value is ? (p can't 
fix a term to the value 1 since we assume f|, is not constant). We denote by x1,...,Zs denote the 
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first s such unassigned variables, according to some canonical ordering of the terms for the k-DNF 
formula of f (there are more than s since otherwise f|, would be expressible as an s-CNF). For 
each such variable x;, let term; be the ?-valued term in which x; appears. Let R; be the operation 
of setting x; to the value that ensures term; is true. We'll map p to 7, = R¡R>2--- Rsp. That is, 
apply Rs to p, then apply Rx-1 to p,---, then apply Rı to p. The crucial insight is that given 7, 
one can deduce term: this is the first term that is true in f|}. One might think that the second 
term that is true in f|» is term but that's not necessarily the case, since the variable x; may have 
appeared several times, and so setting it to Rı may have set other terms to true (it could not have 
set other terms to false, since this would imply that f|, includes an OR of x; and =x;, and hence 
is the constant one function). We thus supply as part of the mapping a string w1 € (0,1,x)* that 
tells us the assignment of the k variables of term, in 79 = Ro---R,p. Given that information we 
can “undo” Rı and move from 7; to 79. Now in 79, terms is the first satisfied term. Continuing 
on this way we see that from 7, (which is an assignment of at least t + s variables) and strings 
w1,...,Ws that are defined as above, we can recover p, implying that we have a one-to-one mapping 
that takes p into an assignment of at least t + s variables and a sequence in (0, 1,34. 


The general case. We now consider the general case, where some terms might have more than 
one unassigned variable in them. We let term, be the first ?-valued term in f|, and let zı be the 
first unassigned variable in term¡. Once again, we have an operation Ry that will make term, true, 
although this time we think of Rı as assigning to all the k variables in term; the unique value that 
makes the term true. We also have an operation Lı assigning a value to xı such that f|z,, cannot 
be expressed by an s — 1-CNF. Indeed, if for both possible assignments to xı we get an s — 1-CNF 
then f|, is an s-CNF. We note that it’s not necessarily the case that x1’s value under Ly pis different 
from its value under R1p, but it is the case that term,’s value is either ? or FALSE under L1p (since 
otherwise f|z,, would be constant). We let termo be the first ?-valued term in f|z,, (note that 
term > term¡) and let x2 be the first unassigned variable in term2. Once again, we have an 
operation Ra such that term) is the first true term in f|r,L,p and operation Lz such that f|L,1,p 
is not a s — 2-CNF. Continuing in this way we come up with operations L1,..., Ls, R1,..., Rs such 
that if we let p; be the assignment L;---L1p (with po = p) then for 1 <i < s: 


e term; is the first ?-valued term in fp, ,. 


e term; is the first true-valued term in f|r,p, ;- 


e L; agrees with p;_; on all variables assigned a value by p;_1. 


e R; agrees with p; on all variables assigned a value by p;. 


For 1 <i < s, define 7; to be R;Rj41---Rsps, and define 7,41 = Ps. We have that term; is 
the first true term in f|,,: indeed, all the operations in 7; do not change variables assigned values 
by pi-1 and there term; is the first ?-valued term. Thus 7; cannot make any earlier term true. 
However, since the last operation applied is R;, term; is true in f|- 

Let 21,...,2s and wi,...,ws be 2s strings in (0,1,x)* defined as follows: z; describes the 
values assigned to the k variables appearing in term; by p;-1 and w; describes the value assigned to 
term,’s variables by 7;+1. Clearly, from term;, z; and the assignment p; one can compute p;-1 and 
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from term;, w; and the assignment 7; one can compute 7;+1. We'll map p to 71 and the sequence 
Z1,-++;%s,W1,---, Ws. Note that 7, does assign values to at least s variables not assigned by p, and 
that from 7, we can find term, (as this is the first true term in f|,,) and then using wı recover 
Tə and continue in this way until we recover the original assignment p. Thus this mapping is a 
one-to-one map from T; s to Z x {0,1,*}°**. m 


13.2 Circuits With “Counters”:ACC 


One way to extend the AC? lowerbounds of the previous section was to define a more general class 
of circuits. What if we allow more general gates? The simplest example is a parity gate. Clearly, 
an AC? circuit provided with parity gates can can compute the parity function. But are there 
still other functions that it cannot compute? Razborov proved the first such lowerbound using his 
Method of Approximations. Smolensky later extended this work and clarified this method for the 
circuit class considered here. 

Normally we think of a modular computation as working with numbers rather than bit, but it 
is sufficient to consider modular gates whose output is always 0/1. 


DEFINITION 13.3 (MODULAR GATES) 
For any integer m, the MOD,, gate outputs 0 if the sum of its inputs is 0 modulo m, and 1 
otherwise. 


DEFINITION 13.4 (ACC) 
For integers m1, mz,..., Mp > 1 we say a language L is in ACC%[m,,mo,...,my)] if there exists a 
circuit family {Cn} with constant depth and polynomial size (and unbounded fan-in) consisting of 
A, V, hand MOD »,,..., MOD, gates accepting L. 

The class ACC? contains every language that is in ACC%(ma, ma,..., mM) for some k > 0 and 
m3,ma,...,mp > 1. 


Good lowerbounds are known only when the circuit has one kind of modular gate. 


THEOREM 13.5 (RAZBOROV,SMOLENSKY) 
For distinct primes p and q, the function MOD, is not in ACC°(q). 


We exhibit the main idea of this result by proving that the parity function cannot be computed 
by an ACC°(3) circuit. 
PROOF: The proof proceeds in two steps. 


Step 1. In the first step, we show (using induction on h) that for any depth h MODs circuit on 
n inputs and size S, there is a polynomial of degree (21)” which agrees with the circuit on 
1 — $/2! fraction of the inputs. If our circuit C has depth d then we set 21 = ni/2d to obtain 


a degree yn polynomial that agrees with C on 1— S/ 2/2 fraction of inputs. 


Step 2 We show that no polynomial of degree yn agrees with MOD 2 on more than 49/50 fraction 
of inputs. 
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Together, the two steps imply that S > gruesa /50 for any depth d circuit computing MOD, 
thus proving the theorem. Now we give details. 
Step 1. Consider a node g in the circuit at a depth h . (The input is assumed to have depth 0.) 
If g(a1,--- , £n) is the function computed at this node, we desire a polynomial g(x1,--- , £n) over 
GF(3) with degree (21)", such that g(x1,...,tn) = 9(21,...,tn) for “most” 21,...,tn € {0,1}. 
We will also ensure that on every input in (0,1)” C GF(3), polynomial g takes a value in (0,1). 
This is without loss of generality since we can just square the polynomial. (Recall that the elements 
of GF(3) are 0,—1,1 and 0? = 0, 1? = 1 and (-1)? = 1.) 

We construct the approximator polynomial by induction. When h = 0 the “gate” is an input 
wire x;, which is exactly represented by the degree 1 polynomial x;. Suppose we have constructed 
approximators for all nodes up to height h — 1 and g is a gate at height h. 


1. If g is a NOT gate, then g = —f; for some other gate fı that is at height h — 1 or less. 
The inductive hypothesis gives an approximator fı for fı. Then we use g = 1 — fı as the 
approximator polynomial for g; this has the same degree as fi. Whenever fi = fı then g = 9, 
so we introduced no new error. 


2. If gis a MODs gate with inputs f1, fo,..., fk, we use the approximation y = NS ae The 
degree increases to at most 2 x (21)’—! < (21)”. Since 0? = 0 and (—1)? = 1, we introduced 
no new error. 


3. If g is an AND or an OR gate, we need to be more careful. Suppose g = NES fi- The naive 
approach would be to replace g with the polynomial I;ezf;. For an OR gate g = a fi De 
Morgan’s law gives a similar naive approximator 1 — ]],_;(1 — f). Unfortunately, both of 
these multiply the degree by k, the fanin of the gate, which could greatly exceed 21. 


The correct solution involves introducing some error. We give the solution for OR; De Mor- 
gan’s law allows AND gates to be handled similarly. 


If g = YES fi, then g = 1 if and only if at least one of the f; = 1. Furthermore, by the random 
subsum principle (see Section ?? in Appendix A) if any of the f; = 1, then the sum (over 
GF(3)) of a random subset of {f;} is nonzero with probability at least 1/2. 


Randomly pick / subsets $1,--- ,S of {1,...,k}. Compute the | polynomials (> jes, ie 
each of which has degree at most twice that of the largest input polynomial. Compute 
the OR of these l terms using the naive approach. We get a polynomial of degree at most 
21 x (21)P=1 = (21)". For any x, the probability over the choice of subsets that this polynomial 
differs from OR(f,,..., fy) is at most x So, by the probabilistic method, there exists a choice 
for the l subsets such that the probability over the choice of x that this polynomial differs from 
OR( Fic e, fr) is at most 4- We use this choice of the subsets to construct the approximator. 


Applying the above procedure for each gate gives an approximator for the output gate of degree 
(21)? where d is depth of the entire circuit. Each operation of replacing the gate by its approximator 
polynomial introduces error on at most 1/2! fraction of all inputs, so the overall fraction of erroneous 
inputs for the approximator is at most S/2!. (Note that errors at different gates may affect each 
other. Error introduced at one gate may be cancelled out by errors at another gate higher up. We 
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are being pessimistic in applying the union bound to upperbound the probability that any of the 
approximator polynomials anywhere in the circuit miscomputes.) 
Step 2. Suppose that a polynomial f agrees with the MOD function for all inputs in a set 
G’ C 0,1”. If the degree of f is bounded by y/n, then we show |G’| < (2)2”. 

Consider the change of variables y; = 1 + z; (mod 3). (Thus 0 — 1 and 1 => —1.) Then, G” 
becomes some subset G of {—1,1}", and f becomes some other polynomial, say g(Yy1,Y2,...,Yn), 
which still has degree yn. Moreover, 


1s II Yi = —1 


MODv3(11,%2,..., Tn) = (2) 


Thus g(y1, Ya, --.,Yn), a degree y/n polynomial, agrees with H?_,y; on G. This is decidedly odd, 


and we show that any such G must be small. Specifically, let Fg be the set of all functions 
49 n 
S:G > {0,1,—-1}. Clearly, |Fa| = 3/6, and we will show |Fa| < 3 (50)? , whence Step 2 follows. 


LEMMA 13.6 
For every S € Fa, there exists a polynomial gs which is a sum of monomials a | [;¿, y; where 
|I| < $+ yn such that gs(x) = S(x) for all x € G. 


PROOF: Let Í: GF(3)” — GF(3) be any function which agrees with S on G. Then $ can be 
written as a polynomial in the variables y;. However, we are only interested in its values on 
(Y1,Y2,---,Yn) € {-1,1}", when y? = 1 and so every monomial Il;ery;* has, without loss of 
generality, r; < 1. Thus Sisa polynomial of degree at most n. Now consider any of its monomial 
terms Ijery; of degree |I| > n/2. We can rewrite it as 


Tierys = Wy YH;e7Yi, (3) 


which takes the same values as g(y1,y2,---,Yn)Ijezy; over {—1,1}". Thus every monomial in S 
has degree at most 5 + yn. W 


To conclude, we bound the number of polynomials whose every monomial with a degree at most 
5 + yn. Clearly this number is #polynomials < Si monomials and 


#monomials < re CA 1 nj NW] < 5 + yn (4) 
El) a 
is5+vn 


Using knowledge of the tails of a binomial distribution (or alternatively, direct calculation), 


4 
< 7 (6 
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13.3 Lowerbounds for monotone circuits 


A Boolean circuit is monotone if it contains only AND and OR gates, and no NOT gates. Such a 
circuit can only compute monotone functions, defined as follows. 


DEFINITION 13.7 
For x,y € [0,1)”, we denote x < y if every bit that is 1 in z is also 1 in y. A function f:{0,1}" = 
{0,1} is monotone if f(x) < f(y) for every x = y. 


REMARK 13.8 
An alternative characterization is that f is monotone if for every input x, changing a bit in x from 
0 to 1 cannot change the value of the function from 1 to 0. 


It is easy to check that every monotone circuit computes a monotone function, and every mono- 
tone function can be computed by a (sufficiently large) monotone circuit. CLIQUE is a monotone 
function since adding an edge to the graph cannot destroy any clique that existed in it. In this 
section we show that the CLIQUE function can not be computed by polynomial (and in fact even 
subexponential) sized monotone circuits: 


THEOREM 13.9 ([RAZ85B, AB87]) 
Denote by CLIQUE,» : {0, 116) — {0,1} be the function that on input an adjacency matrix of an 
n-vertex graph G outputs 1 iff G contains a k-vertex clique. 

There exists some constant e > 0 such that for every k < n'/4, there's no monotone circuit of 
size less than 2V* that computes CLIQUE, n- 


We believe CLIQUE does not have polynomial-size circuits even allowing NOT gates (i.e., that 
NP ¢ P/poly). In fact, a seemingly plausible approach to proving this might be to show that 
for every monotone function f, the monotone circuit complexity of f is polynomially related to 
the general (non-monotone) circuit complexity. Alas, this conjecture was refuted by Razborov 
([Raz85a], see also [Tar88]). 


13.3.1 Proving Theorem 13.9 
Clique Indicators 


To get some intuition why this theorem might be true, lets show that CLIQUE; n can't be computed 
(or even approximated) by subexponential monotone circuits of a very special form. For every 
S C [n], let Cs denote the function on (0, 116) that outputs 1 on a graph G iff the set S is a clique 
in G. We call Cg the clique indicator of S. Note that CLIQUEz n = V ctm], 1S¡=k Cs. We'll now 


prove that CLIQUE; n can't be computed by an OR of less than  VK/20 clique indicators. 

Let Y be the following distribution on n-vertex graphs: choose a set K C [n] with |K| = k at 
random, and output the graph that has a clique on K and no other edges. Let M be the following 
distribution on n-vertex graphs: choose a function c : [n] — [k — 1] at random, and place an edge 
between u and v iff c(u) # c(v). With probability one, CLIQUE, (Y) = 1 and CLIQUE, ¿(N) = 0. 
The fact that CLIQUE, , requires an OR of at least nv /20 clique indicators follows immediately 
from the following lemma: 
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LEMMA 13.10 

Let n be sufficiently large, k < n'/* and S C [n]. Then either Pr[Cs(N) = 1] > 0.99 or Pr[Cs(Y) = 
1] < n—Vk/20 


PROOF: Let £ = Vk — 1/10. If |S| < £ then by the birthday bound, we expect a random f: S = 
[k — 1] to have less than 0.01 collisions and hence by Markov the probability f is one to one is at 
least 0.99. This implies that Pr[Cs(V) = 1] > 0.99. 

If |S| > £ then Pr[Cs(Y) = 1] is equal to the probability that S C K for a random K C [n] 
of size k. This probability is equal to (E8) /(Ẹ) which is at most (,_ AT /10) /(}) which, by the 


formula for the binomial coefficients, is less than ( 2)" ae n~VR/20 (for sufficiently large 
n). E 


Approximation by clique indicators. 
Together with Lemma 13.10, the following lemma implies Theorem 13.9: 


LEMMA 13.11 
Let C be a monotone circuit of size s. Let £ = Vk/10. Then, there exist sets S1,..., Sm with 


m< nvk/20 such that 


ProenwlV Cs,(G) > C(G)] >0.9 (7) 

PraerniY Cs, (G) < C(G)] >0.9 (8) 

(9) 

PROOF: Set £ = Vk/10, p = 10Vklogn and m = (p — 1)é!. Note that m < nV*/20. We can think 
of the circuit C as the sequence of s monotone functions f1,..., fs from {0, 116) to {0,1} where 


each function fẹ is either the AND or OR of two functions fy, fpr for k’,k” < k or is the value 
of an input variable x, for u,v € [n] (Le., fk = Cguvy). The function that C computes is fs. 
We’ll show a sequence of functions f;,..., fs such that each function fẹ is (1) an OR of at most m 
clique indicators Cs,,...,Cs,, with |S;| < £ and (2) fy approximates fy in the sense of (7) and (8). 
We call a function fp satisfying (1) an (¢,m)-function. The result will follow by considering the 
function fs. 

We construct the functions fi,..., fs by induction. For 1 < k < s, if fẹ is an input variable then 
we let fre = fr. If fk = fer V fer then we let fat U fre and if fk = fp A fkr then we let Fi O fpr, where 
the operations U, M will be defined below. We'll prove that for every f,g : {0, 16) — {0,1} (a) if 
f and g are (m, £)-functions then so is f Ug (resp. fg) and (b) Pree,y[fUg (G) < fUg (G)] < 
1/(105) (resp. Praeny|fy (G) < fg (G)] < 1/(105)) and Preezn [fly (G) > fly (G)] < 1/(105) 
(resp. PraezylfMg (G) < fMg (G)] < 1/(105)). The lemma will then follow by showing using 
the union bound that with probability > 0.9 the equations of Condition (b) hold for all fi,..., fs. 
We’ll now describe the two operations L,M. Condition (a) will follow from the definition of the 
operations, while Condition (b) will require a proof. 
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The operation fUg. Let f,g be two (m, £)-functions: that is f = Vj") Cs, and g = VV" Cr, 
(if f or g is the OR of less than m clique indicators we can add duplicate sets to make the number 
m). Consider the function h = Cz, U -++ U Czam where Z; = S; and Zm+; = Tj for 1 < i,j < m. 
The function h is not an (m, £)-function since it is the OR of 2m clique indicators. We make it 
into an (m, £)-function in the following way: as long as there are more than m distinct sets, find 
p subsets Z;,,...,2;, that are in a sunflower formation. That is, there exists a set Z C [n] such 
that for every 1 < j, j’ < p, Zi, N Zij! = Z. Replace the functions Cz, ,..., Cz, in the function h 
with the function Cz. Once we obtain an (m, £)-function h’ we define f Lig to be h’. We won't get 
stuck because of the following lemma (whose proof we defer): 


LEMMA 13.12 (SUNFLOWER LEMMA [ER60]) 
Let Z be a collection of distinct sets each of cardinality at most ¢. If |Z| > (p — 1)*é! then there 
exist p sets Z1,...,Zp € Z and set Z such that Z; N Zj = Z for every 1 < i,j < p. 


The operation fg. Let f,g be two (m, £)-functions: that is f = V7", Cs, and g = Vi"; Cr,. 
Let h be the function Vi<ij<m Cs,ur;- We perform the following steps on h: (1) Discard any 
function Cz for |Z| > @. (2) Reduce the number of functions to m by applying the sunflower 
lemma as above. 


Proving Condition (b). To complete the proof of the lemma, we prove the following four 
equations: 


e PreenylfUg (G) < fUg (G)] < 1/(105). 
If Z C Z1,..., Zp then for every i, Cz,(G) implies that Cz(G) and hence the operation f U g 
can’t introduce any “false negatives” . 


e Preezn[fUg (G) > fg (G)] < 1/(105). 

We can introduce a “false positive” on a graph G only if when we replace the clique indicators 
for a sunflower Z1,..., Zp with the clique indicator for the common intersection Z, it is the 
case that Cz(G) holds even though Cz,(G) is false for every i. Recall that we choose G Er N 
by choosing a random function c : [n] —> [k — 1] and adding an edge for every two vertices u, v 
with c(u) 4 c(v). Thus, we get a false positive if c is one-to-one on Z (we denote this event 
by B) but not one-to-one on Z; for every 1 < i < p (we denote these events by Aj,..., Ap). 
We'll show that the intersection of B and Aj;,...,A, happens with probability at most 27? 
which (by the choice of p) is less than 1/(10m?s). Since we apply the reduction step at most 
m times the equation will follow. 

Since < Vk — 1/10, for every i, Pr[A4;|B] < 1/2 (the probability that there'll be a collision 
on the at most £ elements of Z; \ Z is less than half). Conditioned on B, the events Aj,..., Ap 


are independent, since they depend on the values of c on disjoint sets, and hence we have 
that Pr[A; A---A A, A B] < Pr[A A -+ A A,|B] = I- Pr[4p1B] < 27?. 


e Pree,yl fg (G) < fg (G)] < 1/(108). 
By the distributive law f N g =V,, j(Cs; N Cr,). A graph G in the support of Y consists of a 
clique over some set K. For such a graph Cs, NCr, holds iff S;,7; € K and thus Cs,NCr, holds 
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iff Cs,u7, holds. We can introduce a false negative when we discard functions of the form Cz 


for |Z| > £, but by Lemma 13.10, for such sets Z, Pr[Cz(Y) = 1] < n-Vk/20 < 1/(10sm?). 
The equation follows since we discard at most m? such sets. 


e Praen fg (G) > fMg (G)] < 1/(105). 
Since Csur implies both Cs and Cr, we can't introduce false positives by moving from fMg to 
Vi, j Cs,ur,. We can’t introduce false positives by discarding functions from the OR. Thus, the 
only place where we can introduce false positives is where we replace the clique indicators of 
a sunflower with the clique indicator of the common intersection. We bound this probability 
in the same way as this was done for the U operator. 


Proof of the sunflower lemma (Lemma 13.12). The proof is by induction on £. The case 
£ = 1 is trivial since distinct sets of size 1 must be disjoint. For £ > 1 let M be a maximal 
subcollection of Z containing only disjoint sets. Because of M’s maximality for every Z € Z there 
exists x € UM = UmemM such that x € Z. If |M| > p we're done, since such a collection is 
already a sunflower. Otherwise, since | U M| < (p— 1)£ by averaging there's an x € UM that 
appears in at least a EM fraction of the sets in Z. Let Z1,...,Zx be the sets containing x, and 


note that k > (p — 1)7*(2— 1)!. Thus, by induction there are p sets among the £ — 1-sized sets 
Zi \ {z}, , Zk \ {x} that form a sunflower, adding back x we get the desired sunflower among the 
original sets. Note that the statement (and proof) assume nothing about the size of the universe 
the sets in Z live in. W 


13.4 Circuit complexity: The frontier 


Now we sketch the “frontier” of circuit lowerbounds, namely, the dividing line between what we 
can prove and what we cannot. Along the way we also define multi-party communication, since it 
may prove useful for proving some new circuit lowerbounds. 


13.4.1 Circuit lowerbounds using diagonalization 


We already mentioned that the best lowerbound on circuit size for an NP problem is 4.5n — o(n). 
For PH better lowerbounds are known: one of the exercises in Chapter 6 asked you to show that 
some for every k > 0, some language in PH (in fact in ©) requires circuits of size Q(n*). The 
latter lowerbound uses diagonalization, and one imagines that classes “higher up” than PH should 
have even harder languages. 


Frontier 1: Does NEXP have languages that require super-polynomial size circuits? 


If we go a little above NEXP, we can actually prove a super-polynomial lowerbound: we know 
that MAgxp É P/poly where MApxp is the set of languages accepted by a one round proof with 
an all powerful prover and an exponential time probabilistic verifier. This follows from the fact 
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Figure 13.3: The depth 2 circuit with a symmetric output gate from Theorem 13.13. 


that if MAgxp C P/poly then in particular PSPACE C P/poly. However, by IP = PSPACE 
(Theorem 8.17) we have that in this case PSPACE = MA (the prover can send in one round the 
circuit for computing the prover strategy in the interactive proof). However, by simple padding this 
implies that MAgxp equals the class of languages in exponential space, which can be directly shown 
to not contain P/poly using diagonalization. Interestingly, this lower bound does not relativize (i.e., 
there's an oracle under which MAngxp € P/poly [BFT98]). 


13.4.2 Status of ACC versus P 


The result that PARITY is not in AC? separates NC! from AC?. The next logical step would be 
to separate ACC? from NC!. Less ambitiously, we would like to show even a function in P or NP 
that is not in ACC®. 

The Razborov-Smolenksy method seems to fail when we allow the circuit even two types of 
modular gates, say MOD and MODs. In fact if we allow the bounded depth circuit modular 
gates that do arithmetic mod q, when q is not a prime —a prime power, to be exact— we reach 
the limits of our knowledge. (The exercises ask you to figure out why the proof of Theorem 13.5 
does not seem to apply when the modulus is a composite number.) To give one example, it it is 
consistent with current knowledge that the majority of n bits can be computed by linear size circuits 
of constant depth consisting entirely of MODg¢ gates. The problem seems to be that low-degree 
polynomials modulo m where m is composite are surprisingly expressive [BBR94]. 


Frontier 2: Show Clique is not in ACC®(6). 
Or even less ambitiously: 
Frontier 2.1: Exhibit a language in NEXP that is not in ACC“(6). 


It is worth noting that thus far we are talking about nonuniform circuits (to which Theorem 13.5 
also applies). Stronger lower bounds are known for uniform circuits: Allender and Gore [AG94] 
have shown that a decision version of the Permanent (and hence the Permanent itself) requires 
exponential size “Dlogtime-uniform” ACC? circuits. (A circuit family {Cn} is Dlogtime uniform 
if there exists a deterministic Turing machine M that given a triple (n,g,h) determines in linear 
time —i.e., O(log n) time when g,h < poly(n)— what types of gates g and h are and whether g is 
h’s parent in Cr.) 

But going back to nonuniform ACC®, we wish to mention an alternative representation of 
ACC? circuits that may be useful in further lowerbounds. Let a symmetric gate be a gate whose 
output depends only on the number of inputs that are 1. For example, majority and mod gates 
are symmetric. Yao has shown that ACC? circuits can be simplified to give an equivalent depth 2 
circuits with a symmetric gate at the output (figure ??). Beigel and Tarui subsequently improved 
Yao’s result: 
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THEOREM 13.13 (Yao [YAO90], BEIGEL AND TARUI [BT94]) 
If f € ACC®, then f can be computed by a depth 2 circuit C with a symmetric gate with 


quasipolynomial (i.e., 2log” ") fan-in at the output level and V gates with polylogarithmic fan-in at 
the input level. 


We will revisit this theorem below in Section 13.5.1. 


13.4.3 Linear Circuits With Logarithmic Depth 


When we restrict circuits to have bounded fanin we necessarily need to allow them to have non- 
constant (in fact, (log n)) depth to have any reasonable power. With this in mind, the simplest 
interesting circuit class seems to be one of circuits wth linear size and logarithmic depth. 


Frontier 3: Find an explicit function that cannot be computed by circuits of linear size and 
logarithmic depth. 


(Note that by counting one can easily show that some function on n bits requires superpoly- 
nomial size circuits and hence bounded fan-in circuits with more than logarithmic depth; see the 
exercises on the chapter on circuits. Hence we want to show this for an explicit function, e.g. 
CLIQUE.) 

Valiant thought about this problem in the ’70s. His initial candidates for lowerbounds boiled 
down to showing that a certain graph called a superconcentrator needed to have superlinear size. 
He failed to prove thisand instead ended up proving that such superconcentrators do exist! 

Another sideproduct of Valiant's investigations was the following important lemma concerning 
depth-reduction for such circuits. 


LEMMA 13.14 (VALIANT) 
In any circuit with m edges and depth d, there are km/ logd edges whose removal leaves a circuit 
with depth at most d/2*-!. 


This lemma can be applied as follows. Suppose we have a circuit C of depth clogn with n 
inputs {x1,...,%n} and n outputs [y1,...,Yn), and suppose 2% ~ c/e where e > 0 is arbitrarily 
small. Removing O(n/loglogn) edges from C then results in a circuit with depth at most €logn. 
But then, since C has bounded fan-in, we must have that each output y; is connected to at most 
2clogn — në inputs. So each output y; in C is completely determined by n* inputs and the values 
of the omitted edges. So we have a “dense” encoding for the function f;(11,...,tn) = Yi We do 
not expect this to be the case for any reasonably difficult function. 


13.4.4 Branching Programs 


Just as circuits are used to investigate time requirements of Turing Machines, branching programs 
are used to investigate space complexity. 

A branching program on n input variables 71, £2,..., £n is a directed acyclic graph all of whose 
nodes of nonzero outdegree are labeled with a variable x;. It has two nodes of outdegree zero that 
are labeled with an output value, ACCEPT or REJECT. The edges are labeled by 0 or 1. One of 
the nodes is designated the start node. A setting of the input variables determines a way to walk 
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on the directed graph from the start node to an output node. At any step, if the current node has 
label x;, then we take an edge going out of the node whose label agrees with the value of xi. The 
branching program is deterministic if every nonoutput node has exactly one 0 edge and one 1 edge 
leaving it. Otherwise it is nondeterministic. The size of the branching program is the number of 
nodes in it. The branching program complexity of a language is defined analogously with circuit 
complexity. Sometimes one may also require the branching program to be leveled, whereby nodes 
are arranged into a sequence of levels with edges going only from one level to the next. Then the 
width is the size of the largest level. 


THEOREM 13.15 
If S(n) > logn and L € SPACE(S(n)) then L has branching program complexity at most ce") 
for some constant c > 1. 


PROOF: Essentially mimics our proof of Theorem‘?? that SPACE(S(n)) € DTIME(20(6™)), 
The nodes of the branching program correspond to the configurations of the space-bounded TM, 
and it is labeled with variable x; if the configuration shows the TM reading the ith bit in the input. 
E 


Of course, a similar theorem is true about NDTMs and nondeterministic branching program 
complexity. 


Frontier 4: Describe a problem in P (or even NP) that requires branching programs of size greater 
than n!*¢ for some constant e > 0. 

There is some evidence that branching programs are more powerful than one may imagine. For 
instance, branching programs of constant width (reminiscent of a TM with O(1) bits of memory) 
seem inherently weak. Thus the next result is unexpected. 


THEOREM 13.16 (BARRINGTON [?]) 
A language has polynomial size, width 5 branching programs iff it is in NC. 


13.5 Approaches using communication complexity 


Here we outline a concrete approach (rather, a setting) in which better lowerbounds may lead to a 
resolution of some of the questions above. It relates to generalizations of communication complexity 
introduced earlier. Mostly we will use multiparty communication complexity, though Section 13.5.4 
will use communication complexity of a relation. 


13.5.1 Connection to ACC? Circuits 


Suppose f(x1,..., £k) has a depth-2 circuit with a symmetric gate with fan-in N at the output and 
A gates with fan-in k—1 at the input level (figure 2). The claim is that f’s k-party communication 
complexity is at most klog N. (This observation is due to Razborov and Wigderson [RW93]). To 
see the claim, first partition the A gates amongst the players. Each bit is not known to exactly one 
player, so the input bits of each A gate are known to at least one player; assign the gate to such a 
player with the lowest index. Players then broadcast how many of their gates output 1. Since this 
number has at most log N bits, the claim follows. 
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Figure 13.4: If f is computed by the above circuit, then f has a k-party protocol of complexity klog N. 


Our hope is to employ this connection with communication complexity in conjunction with 
Theorem 13.13 to obtain lower bounds on ACC? circuits. For example, note that the function in 
Example ?? above cannot have k < logn/4. However, this is not enough to obtain a lower bound 
on ACC? circuits since we need to show that k is not polylogarithmic to employ Theorem 13.13. 
Thus a strengthening of the Babai Nisan Szegedy lowerbound to Q(n/poly(k)) for say the CLIQUE 
function would close Frontier 2. 


13.5.2 Connection to Linear Size Logarithmic Depth Circuits 


Suppose that f : {0,1}" x {0,1}!°8" — {0,1}” has bounded fan-in circuits of linear size and 
logarithmic depth. If f(x, j,i) denotes the ith bit of f(x,j), then Valiant’s Lemma implies that 
F(x, 3,1) has a simultaneous 3-party protocol—that is, a protocol where all parties speak only once 
and write simultaneously on the blackboard (i.e., non-adaptively)—where, 


e (x,7) player sends n/ log log n bits; 

e (1,1) player sends n* bits; and 

e (i,j) player sends O(log n) bits. 
So, if we can show that a function does not have such a protocol, then we would have a lower bound 
for the function on linear size logarithmic depth circuits with bounded fan-in. 


Conjecture: The function f(x, j,i) = £jgi, where j O i is the bitwise xor, is conjectured to be 
hard, i.e., f should not have a compact representation. 


13.5.3 Connection to branching programs 


The notion of multiparty communication complexity (at least the “number on the forehead” model 
discussed here) was invented by Chandra Furst and Lipton [?] for proving lowerbounds on branching 
programs, especially constant-width branching programs discussed in Section ?? 


13.5.4 Karchmer-Wigderson communication games and depth lowerbounds 


The result that PARITY is not in AC? separates NC! from AC®°. The next step would be to 
separate NC? from NC!. (Of course, ignoring for the moment the issue of separating ACC? from 
NC!.) Karchmer and Wigderson [KW90] described how communication complexity can be used 
to prove lowerbounds on the minimum depth required to compute a function. They showed the 
following result about monotone circuits, which we will not prove this result. 


THEOREM 13.17 
Detecting whether a graph has a perfect matching is impossible with monotone circuits of depth 
O(log n) 
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However, we do describe the basic Karchmer- Wigderson game used to prove the above result, 
since it is relevant for nonmonotone circuits as well. For a function f:(0,1)” — {0,1} this game 
is defined as follows. 


There are two players, ZERO and ONE. Player ZERO receives an input x such that f(x) = 0 
and Player ONE receives an input y such that f(y) = 1. They communicate bits to each other, 
until they can agree on ani € {1,2,...,n} such that x; 4 yi. 


The mechanism of communication is defined similarly as in Chapter 12; there is a protocol that 
the players agree on in advance before receiving the input. Note that the key difference from the 
scenario in Chapter 12 is that the final answer is not a single bit, and furthermore, the final answer 
is not unique (the number of acceptable answers is equal to the number of bits that x, y differ on). 
Sometimes this is described as computing a relation. The relation in this case consists of all triples 
(a, 4,4) such that f(z) =0, f(y) =1 and a; 4 yi. 

We define Cxw(f) as the communication complexity of the above game; namely, the maximum 
over all x € f~1(0),y € F7*(1) of the number of bits exchanged in computing an answer for x,y. The 
next theorem shows that this parameter has a suprising alternative characterization. It assumes 
that circuits don't have NOT gates and instead the NOT gates are pushed down to the inputs 
using De Morgan’s law. (In other words, the inputs may be viewed as z1, %2,...,Un,%1,%2,---,Xn-) 
Furthermore, AND and OR gates have fanin 2. (None of these assumptions is crucial and affects 
the theorem only marginally.) 


THEOREM 13.18 ([KW90]) 
Cxw(f) is exactly the minimum depth among all circuits that compute f. 


PROOF: First, we show that if there is a circuit C of depth K that computes f then Cxw(f) < K. 
Each player has a copy of C, and evaluates this circuit on the input given to him. Of course, it 
ealuates to 0 for Player ZERO and to 1 for Player ONE. Suppose the top gate is an OR. Then 
at least one of the two incoming wires to this gate must be 1, and in the first round, Player ONE 
sends one bit communicating which of these wires it was. Note that this wire is 0 for Player ZERO. 
In the next round the players focus on the gate that produced the value on this wire. (If the top 
gate is an AND on the other hand, then in the first round Player ZERO speaks, conveying which 
of the two incoming wires was 0. This wire will be 1 for Player ONE.) This goes on and the players 
go deeper down the circuit, always maintaining the invariant that the current gate has value 1 for 
Player ONE and 0 for Player ZERO. Finally, after at most K steps they arrive at an input bit. 
According to the invariant being maintained, this bit must be 1 for Player ONE and 0 for Player 
ZERO. Thus they both know an index ¿ that is a valid answer. 

For the reverse direction, we have to show that if Cxw(f) = K then there is a circuit of depth 
at most K that computes f. We prove a more general result. For any two disjoint nonempty 
subsets A C f71(0) and B C f7*(1), let Cxw(A, B) be the communication complexity of the 
Karchmer-Wigderson game when x always lies in A and y in B. We show that there is a circuit 
of depth Cgw (A, B) that outputs 0 on every input from A and 1 on every input from B. Such a 
circuit is called a distinguisher for sets A, B. The proof is by induction on K = Cxw(A, B). The 
base case K = 0 is trivial since this means the players do not have to communicate at all to agree 
on an answer, say i. Hence x; Æ yi for all x € A,y € B, which implies that either (a) x; = 0 for 
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every x € A and y; = 0 for every y € B or (b) x; = 1 for every x € A and y; = 1 for every y € B. 
In case (a) we can use the depth 0 circuit x; and in case (b) we can use the circuit 7; to distinguish 
A, B. 

For the inductive step, suppose Cxw(A, B) = K, and at the first round Player ZERO speaks. 
Then A is the disjoint union of two sets Ag, A¡ where A; is the set of inputs in A for which Player 
ZERO sends bit b. Then Cxyw(4, B) < K — 1 for each b, and the inductive hypothesis gives a 
circuit Cy, of depth at most K — 1 that distinguishes Ap, B. We claim that Co A Ci distinguishes 
A, B (note that it has depth at most K). The reason is that Co(y) = Ci(y) = 1 for every y € B 
whereas for every x € A, Co(x) A Ci (x) = 0 since if x € A, then C(x) = 0. MW 


Thus we have the following frontier. 
Frontier 5: Show that some function f in P (or even NEXP!) has Cxw(f) = Q(log n log log n). 


Karchmer, Raz, and Wigderson [KRW95] describe a candidate function that may work. It uses 
the fact a function on k bits has a truth table of size 2", and that most functions on k bits are hard 
(e.g., require circuit size 0(2"/k), circuit depth Q(k), etc.). They define the function by assuming 
that part of the n-bit input encodes a very hard function, and this hard function is applied to the 
remaining input in a “tree” fashion. 

For any function g: {0,1}* — {0,1} and s > 1 define g°*: {0,1}* — {0,1} as follows. If s = 1 
then g°* = g. Otherwise express the input x € {0, py as 111213 -Ep Where each x; € (0, pea 
and define 


g*( (1113 ds Tk) = age (21) 9D (2) Se gD (axp)). 


Clearly, if g can be computed in depth d then g°* can be computed in depth sd. Furthermore, if 
one fails to see how one could reduce the depth for an arbitrary function. 

Now we describe the KRW candidate function f :{0,1}" — (0,1). Let k = [log }] and s be 
the largest integer such that k* < n/2 (thus s = O( eee a):) For any n-bit input 2, let gy be the 
function whose truth table is the first 2} bits of x. Let z| be the string of the last k* bits of x. 
Then 


According to our earlier intuition, when the first 2* bits of x represent a really hard function —as 
log? n 
log log n ) ` 


they must for many choices of the input— then g*(x|2) should require depth Q(sk) = Q( 
Of course, proving this seems difficult. 

This type of complexity questions, whereby we are asking whether s instances of a problem are 
s times as hard as a single instance, are called direct sum questions. Similar questions have been 
studied in a variety of computational models, and sometimes counterintuitive results have been 
proven for them. One example is that by a counting argument there exists an n x n matrix A over 
{0,1}, such that the smallest circuit computing the linear function v + Av for v € {0,1}” is of 
size O(n”). However, computing this function on n instances v1,...,Un can be done significantly 
faster than n* steps using fast matrix multiplication [Str69] (the current record is roughly O(n?38) 
[CW90]). 
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Chapter notes and history 


Shannon defined circuit complexity, including monotone circuit complexity, in 1949. The topic was 
studied in Russia since the 1950s. (See Trakhtenbrot [Tra84] for some references.) Savage [Sav72 
was the first to observe the close relationship between time required to decide a language on a 
TM and its circuit complexity, and to suggest circuit lowerbounds as a way to separate complexity 
classes. A burst of results in the 1980s, such as the separation of P from AC? [FSS84, Ajt83 
and Razborov's separation of monotone NP from monotone P/poly [Raz85b] raised hopes that a 
resolution of P versus NP might be near. These hopes were dashed by Razborov himself [Raz89 
when he showed that his method of approximations was unlikely to apply to nonmonotone circuits. 
Later Razborov and Rudich [RR97] formalized what they called natural proofs to show that all 
lines of attack considered up to that point were unlikely to work. (See Chapter 22.) 


Our presentation in Sections 13.2 and 13.3 closely follows that in Boppana and Sipser’s excellent 
survey of circuit complexity [BS90], which is still useful and current 15 years later. (It omits 
discussion of lowerbounds on algebraic circuits; see [Raz04] for a recent result.) 

Hastad’s switching lemma [Has86] is a stronger form of results from[FSS84, Ajt83, Yao85). 
The Razborov-Smolensky method of using approximator polynomials is from [Raz87], strength- 
ened in[Smo87]. Valiant’s observations about superlinear circuit lowerbounds are from a 1975 
paper [Val75] and an unpublished manuscript—lack of progress on this basic problem gets more 
embarrassing by the day!. 

The 4.5n — o(n) lowerbound on general circuits is from Lachish-Raz [LRO1]. 


Exercises 


$1 Suppose that f is computable by an AC 0 circuit C of depth d and size S. Prove that f is 
computable by an AC 0 circuit C” of size 105 and depth d that does not contain NOT gates 
but instead has n additional inputs that are negations of the original n inputs. 


*“u01ye39u 
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$2 Suppose that f is computable by an AC 0 circuit C of depth d and size S. Prove that f is 
computable by an ACO C” circuit of size (109)? and depth d where each gate has fanout 1. 


k 
§3 Prove that for t > n/2, a at) (=) . Use this to complete the proof of Lemma 13.2 
(Section 13.1.2). 


§4 Show that ACC? € NC!. 


85 Identify reasons why the Razborov-Smolensky method does not work when the circuit has 
modm gates, where m is a composite number. 


86 Show that representing the OR of n variables 11,72,...,7, exactly with a polynomial over 
GF(q) where q is prime requires degree exactly n. 
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87 The Karchmer-Wigderson game can be used to prove upperbounds, and not just lowerbounds. 
Show using this game that PARITY and MAJORITY are in NC!. 


$8 Show that if a language is computed by a polynomial-size branching program of width 5 then 
it is in NC’. 


§9 Prove Valiant’s Lemma (Lemma 13.14). 
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Chapter 14 


Algebraic computation models 


The Turing machine model captures computations on bits (equivalently, integers), but it does not 
always capture the spirit of algorithms which operate on, say the real numbers R or complex num- 
bers C. Such algorithms arise in a variety of applications such as numerical analysis, computational 
geometry, robotics, and symbolic algebra. A simple example is Newton’s method for finding roots 
of a given real-valued function function f. It iteratively produces a sequence of candidate solutions 
£0, £1, £2, ..., € R where 241 = xi — f(2,)/f'(x;). Under appropriate conditions this sequence can 
be shown to converge to a root of f. 

Of course, a perfectly defensible position to take is that even the behavior of such algorithms 
should be studied using TMs, since they will be run on real-life computers, which represent real 
numbers using finite precision. In this chapter though, we take a different approach and study 
models which do allow arithmetic operations on real numbers (or numbers from fields other than 
R). Such an idealized model may not be implementable, strictly speaking, but it provides a useful 
approximation to the asymptotic behavior as computers are allowed to use more and more precision 
in their computations. Furthermore, one may be able to prove nontrivial lowerbounds for these 
models using techniques from well-developed areas of mathematics such as algebraic geometry and 
topology. (By contrast, boolean circuit lowerbounds have proven very difficult.) 

However, coming up with a meaningful, well-behaved model of algebraic computation is not an 
easy task, as the following example suggests. 


EXAMPLE 14.1 (PITFALLS AWAITING DESIGNERS OF SUCH MODELS) 

A real number can encode infinite amount of information. For example, a single real number is 
enough to encode the answer to every instance of SAT (or any other language, in general). Thus, 
a model that can store any real number with infinite precision may not be realistic. Shamir has 
shown how to factor any integer n in poly(log n) time on a computer that can do real arithmetic 
with arbitrary precision. 


The usual way to avoid this pitfall is to restrict the algorithms’ ability to access individual 
bits (e.g., the machine may require more than polynomial time to extract a particular digit from 
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a real number). Or, sometimes (as in case of Algebraic Computation Trees) it is OK to consider 
unrealistically powerful models since the goal is to prove nontrivial lowerbounds —say, superlinear 
or quadratic— rather than arbitrary polynomial lowerbounds. After all, lowerbounds for unrealis- 
tically powerful models will apply to more realistic (and weaker) models as well. 

This chapter is a sketchy introduction to algebraic complexity. It introduces three algebraic 
computation models: algebraic circuits, algebraic computation trees, and algebraic Turing Ma- 
chines. The algebraic TM is closely related to the standard Turing Machine model and allows 
us to study similar questions for arbitrary fields — including decidability and complexity—that 
we earlier studied for strings over {0,1}. We introduce an undecidable problem (namely, deciding 
membership in the Mandelbrot set) and and an NP-complete problem (decision version of Hilbert’s 
Nullstellensatz) in this model. 


14.1 Algebraic circuits 


An algebraic circuit over a field F is defined by analogy with a boolean circuit. It consists of a 
directed acyclic graph. The leaves are called input nodes and labeled 71, £2,..., £n, except these 
take values in a field F' rather than boolean variables. There are also special input nodes, labeled 
with the constants 1 and —1 (which are field elements). Each internal node, called a gate, is labeled 
with one of the arithmetic operations {+, x} rather than with the boolean operations V, A, = used 
in boolean circuits. There is only output node. We restrict indegree of each gate to 2. The size of 
the circuit is the number of gates in it. One can also consider algebraic circuits that allow division 
(+) at the gates. One can also study circuits that have access to “constants” other than 1; though 
typically one assumes that this set is fixed and independent of the input size n. Finally, as in the 
boolean case, if each gate has outdegree 1, we call it an arithmetic formula. 

A gate’s operation consists of performing the operation it is labeled with on the numbers present 
on the incoming wires, and then passing this output to all its outgoing wires. After each gate has 
performed its operation, an output appears on the circuit’s lone output node. Thus the circuit may 
be viewed as a computing a function f(11,12,...,Tn) of the input variables, and simple induction 
shows that this output function is a (multivariate) polynomial in x1, £2,..., £n. If we allow gates to 
also be labeled with the division operation (denoted “+”) then the function is a rational function 
of 11,...,Tn, in other words, functions of the type fi(11,t2,...,Tn)/fo[t1,..., Tn) where fi, fo 
are polynomials. Of course, if the inputs come from a field such as R, then rational functions can 
be used to approximate —via Taylor series expansion —all “smooth” real-valued functions. 

As usual, we are interested in the asymptotic size (as a function of n) of the smallest family 
of algebraic circuits that computes a family of polynomials {fp} where fn is a polynomial in n 
variables. The exercises ask you to show that circuits over GF(2) (with no +) are equivalent to 
boolean circuits, and the same is true for circuits over any finite field. So the case when F is infinite 
is usually of greatest interest. 


EXAMPLE 14.2 

The discrete fourier transform of a vector a = (ao, a1, . . . ,an—1) where a; € C is vector M-a, where 
M is a fixed n x n matrix whose (i, j) entry is wt where w is an nth root of 1 (in other words, a 
complex number satisfying w” = 1). 
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Interpreting the trivial algorithm for matrix-vector product as an arithmetic circuit, one obtains 
an algebraic formula of size O(n”). Using the famous fast fourier transform algorithm, one can 
obtain a smaller circuit (or formula??; CHECK) of size O(nlogn). 

STATUS OF LOWERBOUNDS?? 


EXAMPLE 14.3 
The determinant of an n x n matrix X = (X;¿) is 


det(X) = y [] rio, (1) 


dESn i=1 


where Sn is the set of all n! permutations on {1,2,...,2}. This can be computed using the familiar 
Gaussian elimination algorithm. Interpreting the algorithm as a circuit one obtains an arithmetic 
circuit of size O(n). Using the NC? algorithm for Gaussian elimination, one obtains an arithmetic 
formula of size 200871) No matching lowerbounds are known for either upperbound. 


The previous example is a good illustration of how the polynomial defining a function may have 
exponentially many terms —in this case n!—but nevertheless be computable with a polynomial-size 
circuit (as well as a subexponential-size formula). 

By contrast, no polynomial-size algebraic circuit is conjectured to exist for the permanent 
function, which at first sight seems is very similar to the determinant but as we saw in Section ??, 
is #P-complete. 


n 
permanent(X) = y ia TEZO (2) 
TESn 4=1 

The determinant and permanent functions also play a vital role in the world of algebraic circuits, 
since they are complete problems for two important classes. To give the definition, we need the 
notion of degree of a multivariate polynomial, namely, the minimum d such that each monomial 
term II, c” satisfies )¿d¡ < d. A family of polynomials in x1,12,...,tn is poly-bounded if the 
degree is at most O(n) for some constant c > 0. 
DEFINITION 14.4 (AlgP) 
The class AlgP is the class of polynomials of polynomial degree that are computable by arithmetic 
formulae (using no +) of polynomial size. 


DEFINITION 14.5 (ALGNP) 
AlgNP is the class of polynomials of polynomial degree that are definable as 
f (£1, £2, eer) = y iD led 
e€{0,1}™-” 


where gn € AlgP and m is polynomial in n. 
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DEFINITION 14.6 (PROJECTION REDUCTION) 

A function f(11,...,tn) is a projection of a function g(y1, y2,---,Ym) if there is a mapping o from 

(41, Y2)---,Ym) to {0, 1, £1, £2,- . ., En} such that f(21,22,..., Un) = glo (y1), 0 (Ya), . . , (Ym)). 
We say that f is projection-reducible to g if f is a projection of g. 


THEOREM 14.7 (VALIANT) 
Every polynomial on n variables that is computable by a circuit of size u is projection reducible to 
the Determinant function (over the same field) on u+ 2 variables. 


Every function in AlgNP is projection reducible to the Permanent function (over the same 
field). 


14.2 Algebraic Computation Trees 


An algebraic computation tree is reminiscent of a boolean decision tree (Chapter ??) but it computes 
a boolean-valued function f:R” — {0,1}. Consider for example the ELEMENT DISTINCTNESS 
problem of deciding, given n numbers 21, £2,..., £n, whether any two of them are the same. To 
study it in the decision tree model, we might study it by thinking of the input as a matrix of size n? 
where the (i, j) entry indicates whether or or not x; > £j or £; = xj or £i < £j. But one can also 
study it as a problem whose input is a vector of n real numbers. Consider the trivial algorithm in 
either viewpoint: sort the numbers in O(nlogn) time and then check if any two adjacent numbers 
in the sorted order are the same. Is this trivial algorithm actually optimal? This question is 
still open, but one can prove optimality with respect to a more restricted class of algorithms that 
includes the above trivial algorithm. 

Recall that comparison-based sorting algorithms only ask questions of the type “Is 2; > x;?”, 
which is the same as asking whether x; — 2; > 0. The left hand side term of this last inequality is a 
linear function. Other algorithms may use more complicated functions. In this section we consider 
a model called Algebraic Computation Trees, where we examine the effect of allowing a) the use of 
any polynomial function and b) the introduction of new variables together with the ability to ask 
questions about them. 


DEFINITION 14.8 (ALGEBRAIC COMPUTATION TREE) 

An Algebraic Computation Tree is a way to represent a function f:R” — {0,1} by showing how 
to compute f(x1,22,...,2%n) for any input vector (£1, £2,..., Zn). It is a complete binary tree that 
describes where each of the nodes has one of the following types: 


e Leaf labeled “Accept” or “Reject”. 


e Computation node v labeled with y,, where y = Yu © Yw and Yu, Yw are either one of 
{x1,£2,...,%n} or the labels of ancestor nodes and the operator o is in {+, —, x, +, e 


e Branch node with out-degree 2. The branch that is taken depends on the evaluation of some 
condition of the type Yu = 0 or Yu > 0 or Yu < 0 where yu is either one of (11, 12,..., Zn} Or 
the labels of an ancestor node in the tree. 
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Figure 14.1: An Algebraic Computation Tree 


Figure unavailable in pdf file. 


Figure 14.2: A computation path p of length d defines a set of constraints over the n input variables x; and d 
additional variables y;, which correspond to the nodes on p. 


The computation on any input (£1, 2%2,..., Tp) follows a single path the root to a leaf, evaluating 
functions at internal nodes (including branch nodes) in the obvious way. The complexity of the 
computation on the path is measured using the following costs (which reflect real-life costs to some 
degree): 


e +,— are free. 
e x,>+,,/ are charged unit cost. 


The depth of the tree is the maximum cost of any path in it. 


A fragment of an algebraic decision tree is shown in figure 14.1. The following examples illustrate 
some of the languages (over real numbers) whose complexity we want to study. 


EXAMPLE 14.9 

[Element Distinctness Problem] Given n numbers z1, £2, .. ., Zn we need to determine whether they 
are all distinct. This is equivalent to the question whether |];_;(%; — xj) # 0. As indicated earlier, 
this can be computed by a tree of depth O(nlogn) whose internal nodes only compute functions 
of the type x; — 25. 


EXAMPLE 14.10 
[Real number version of SUBSET SUM] Given a set of n real numbers X = [11,12,...,T,j we ask 


whether there is a subset S C X such that Dies x= 1. 


Of course, a tree of depth d could have 2% nodes, so a small depth decision tree does not always 
guarantee an efficient algorithm. This is why the following theorem (which we do not prove) does 
not have any implication for P versus NP. 


THEOREM 14.11 
The real number version of SUBSET SUM can be solved using an algebraic computation tree of depth 


O(n?). 
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This theorem suggests that Algebraic Computation Trees are best used to investigate lower- 
bounds such as nlog n or n?. To prove lowerbounds for a function f, we will use the topology of the 
sets f-'(1) and f7*(0), specifically, the number of connected components. In fact, we will think of 
any function f:R” — R as being defined by a subset W C R”, where W = f71(1). 


DEFINITION 14.12 
Let W CR”. The algebraic computation tree complexity of W is 


C(W) = min {depth of C} 
computation 
trees C for W 


DEFINITION 14.13 (CONNECTED COMPONENTS) 
A set S CR” is connected if for all x,y € S there is path p that connects x and y and lies entirely 
in S. For S CR” we define #(S) to be the number of connected components of S. 


THEOREM 14.14 
Let W = {(21,.--,2n)| Tz; (vi — 25) # OF. Then, 


#(W) >n! 
PROOF: For each permutation o let 
Ws = {(x1, da sie) | Xe (1) < Le (2) Ús E 


That is, let W, be the set of n-tuples (z1,..., £n) to which o gives order. It suffices to prove for 
all o 4 o that the sets W, and Wy are not connected. 

For any two distinct permutations o and o”, there exist two distinct 7,7 with 1 < i,j < n, such 
that 07*(6) < 075) but o™t(i) > o™}(j). Thus, in W, we have X; — X; > 0 while in Wy we 
have X; — X; > 0. Consider any path from Ws to Ws. Since X; — X; has different signs at the 
endpoints, the intermediate value principle says that somewhere along the path this term must 
become 0. Definition 14.13 then implies that W, and W, cannot be connected. W 


The connection between the two parameters we have defined thus far is the following theo- 
rem, whose proof will use a fundamental theorem of topology. It also implies, using our obser- 
vation above, that the algebraic computation tree complexity of ELEMENT DISTINCTNESS is 
Q(og(n!)) = Q(n logn). 


THEOREM 14.15 (BEN-OR) 


C(W) =0( log (max {#(W), #(R" — W))) —n) 


This theorem is proved in two steps. First, we try to identify the property of functions with 
low decision tree complexity: they can be defined using a “few” systems of equations. 
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If f:R” — {0,1} has a decision tree of depth d then f7*(1) (and also f7*(0)) is a union of at most 
2¢ sets C1,C2,..., where C; is the set of solutions to some algebraic system of up to d equations 
of the type 


Dildo Yd; £1,- - -, En) PAD; 


where p; for i < d is a degree 2 polynomial, & is in {<,>,=,4}, and y1,..., ya are new variables. 
(Rabinovitch’s Trick) Additionally, we may assume without loss of generality (at the cost of 
doubling the number of y;’s) that there are no Æ constraints in this system of equations. 


PROOF: The tree has 2? leaves, so it suffices to associate a set with each leaf. This is simply the set 

of (11,12,..., Zn) that end up at that leaf. Associate variables y1, y2,...,yq with the d tree nodes 

appearing along the path from root to that leaf. For each tree nodes associate an equation with it 

in the obvious way (see figure 14.2). For example, if the node computes Yy = Yu + Yw then it implies 

the constraint YvYw — Yu = 0. Thus any (11,1%2,..., Un) that end up at the leaf is a vector with an 

associated value of y1, y2,...,Yq such that the combined vector is a solution to these d equations. 
To replace the “4” constraints with “=” constraints we take a constraint like 


pila, site 1 Ym) = 0, 


introduce a new variable z; and impose the constraint 


qilY1,---Ym, Zi) =1— 24pilYz, -- -, Ym) = 0. 


(This transformation holds for all fields.) Notice, the maximum degree of the constraint remains 
2, because the trick is used only for the branch y, 4 0 which is converted to 1 — zyy, = 0. 
a 


REMARK 14.17 
We find Rabinovitch’s trick useful also in Section 14.3.2 where we prove a completeness result for 
Hilbert’s Nullstellensatz. 

Another version of the trick is to add the constraint 


DACIE poga Ym) > 0, 


which doubles the degree and does not hold for all fields (e.g., the complex numbers). 


Thus we need some result about the number of connected components of the set of solutions to 
an algebraic system. The following is a central result in mathematics. 
THEOREM 14.18 (SIMPLE CONSEQUENCE OF MILNOR-THOM) 
If S C R” is defined by degree d constraints with m equalities and h inequalities then 
#(S) < d(2d— 19041 


REMARK 14.19 
Note that the above upperbound is independent of m. 
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Figure 14.3: Projection can merge but not add connected components 


Now we can prove Ben-Or’s Theorem. 


PROOF: (Theorem 14.15) Suppose that the depth of a computation tree for W is d, so that there 
are at most 2% leaves. We will use the fact that if S C R” and Sly is the set of points in S with 
their n — k coordinates removed (projection on the first k coordinates) then #(S|,) < #(S) (figure 
14.3). 


For every leaf there is a set of degree 2 constraints. So, consider a leaf £ and the corresponding 
constraints Cg, which are in variables 71,...,2%n,41,--.,Yqa. Let We C R” be the subset of inputs 
that reach @ and Sy C R"*@ the set of points that satisfy the constraints Cz. Note that We = Cy|n 
i.e., We is the projection of Ce onto the first n coordinates. So, the number of connected components 
in Wọ is upperbounded by #(Cy). By Theorem 14.18 #(Cy) < 2-3"+¢! < 3"+4. Therefore the 
total number of connected components is at most 243"+4, so d > log(#(W)) — O(n). By repeating 
the same argument for R” — W we have that d > log(#(R"” — W)) — O(n). E 


143 The Blum-Shub-Smale Model 


Blum, Shub and Smale introduced Turing Machines that compute over some arbitrary field K (e.g., 
K = R, C, Z2). This is a generalization of the standard Turing Machine model which operates over 
the ring Z2. Each cell can hold an element of K, Initially, all but a finite number of cells are “blank.” 
In our standard model of the TM, the computation and branch operations can be executed in the 
same step. Here we perform these operations separately. So we divide the set of states into the 
following three categories: 


e Shift state: move the head to the left or to the right of the current position. 
e Branch state: if the content of the current cell is a then goto state qı else goto state qo. 


e Computation state: replace the contents of the current cell with a new value. The machine 
has a hardwired function f and the new contents of the cell become a — f(a). In the standard 
model for rings, f is a polynomial over K, while for fields f is a rational function p/q where 
p,q are polynomials in K[x] and q 4 0. In either case, f can be represented using a constant 
number of elements of K. 


e The machine has a single “register” onto which it can copy the contents of the cell currently 
under the head. This register?s contents can be used in the computation. 


In the next section we define some complexity classes related to the BSS model. As usual, the 
time and space complexity of these Turing Machines is defined with respect to the input size, which 
is the number of cells occupied by the input. 
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REMARK 14.20 
The following examples show that some modifications of the BSS model can increase significantly 
the power of an algebraic Turing Machine. 


e If we allow the branch states to check, for arbitrary real number a, whether a > 0 (in 
other words, with arbitrary precision) the model becomes unrealistic because it can decide 
problems that are undecidable on the normal Turing machine. In particular, such a machine 
can compute P/poly in polynomial time; see Exercises. (Recall that we showed that P/poly 
contains undecidable languages.) If a language is in P/poly we can represent its circuit family 
by a single real number hardwired into the Turing machine (specifically, as the coefficient of 
of some polynomial p(x) belonging to a state). The individual bits of this coefficient can be 
accessed by dividing by 2, so the machine can extract the polynomial length encoding of each 
circuit. Without this ability we can prove that the individual bits cannot be accessed. 


e If we allow rounding (computation of |x|) then it is possible to factor integers in polynomial 
time, using some ideas of Shamir. (See exercises.) 


Even without these modifications, the BSS model seems more powerful than real-world com- 
puters: Consider the execution of the operation z — a? for n times. Since we allow each cell to 
store a real number, the Turing machine can compute and store in one cell (without overflow) the 
number x?” in n steps. 


14.3.1 Complexity Classes over the Complex Numbers 


Now we define the corresponding to P and NP complexity classes over C: 


DEFINITION 14.21 (Pc,NPc) 

Pc is the set of languages that can be decided by a Turing Machine over C in polynomial time. 
NP is the set of languages L for which there exists a language Lo in Pe, such that an input x is 
in L iff there exists a string (y1,...,Yne) in C™ such that (x,y) is in Lo. 


The following definition is a restriction on the inputs of a TM over C. These classes are useful 
because they help us understand the relation between algebraic and binary complexity classes. 


DEFINITION 14.22 (0-1-NPc) 


0-1-NPc = {LN {0, 1 | LE NPc} 


Note that the input for an NP machine is binary but the nondeterministic “witness” may 
consist of complex numbers. Trivially, 3SAT is in 0-I1-NPco: even though the “witness” consists of 
a string of complex numbers, the machine first checks if they are all 0 or 1 using equality checks. 
Having verified that the guess represents a boolean assignment to the variables, the machine con- 
tinues as a normal Turing Machine to verify that the assignment satisfies the formula. 


It is known that 0-1-NPc C PSPACE. In 1997 Koiran proved that if one assumes the Riemann 
hypothesis, then 0-1-NPc € AM[2]. Recall that AM[2] is BP - NP so Koiran’s result suggests 
that 0-1-NPc may not be much bigger than NP. 
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Figure 14.4: Tableau of Turing Machine configurations 


14.3.2 Hilbert's Nullstellensatz 


The language HN dc is defined as the decision version of Hilbert’s Nullstellensatz over C. The input 
consists of m polynomials p; of degree d over z1,...,£n. The output is “yes” iff the polynomials 
have a common root aj,...,@,. Note that this problem is general enough to include SAT. We 
illustrate that by the following example: 


gVyVze(1-—2z)(1-—y)(1-z) =0. 


Next we use this fact to prove that the language 0-1-HNc (where the polynomials have 0-1 coeffi- 
cients) is complete for 0-1-NPc. 


THEOREM 14.23 (BSS) 
0-1-HNc is complete for 0-1-NPc. 


PROOF: (Sketch) It is straightforward to verify that 0-1-HNc is in 0-1-NPc. To prove the hard- 
ness part we imitate the proof of the Cook-Levin theorem; we create a computation tableau and 
show that the verification is in 0-1-HNc. 


To that end, consider the usual computation tableau of a Turing Machine over C and as in the 
case of the standard Turing Machines express the fact that the tableau is valid by verifying all the 
2 x 3 windows, i.e., it is sufficient to perform local checks (Figure 14.4). Reasoning as in the case of 
algebraic computation trees (see Lemma 14.16) we can express these local checks with polynomial 
constraints of bounded degree. The computation states c — q(a,b)/r(a,b) are easily handled by 
setting p(c) = q(a,b) — cr(a,b). For the branch states p(a,b) 4 0 we can use Rabinovitch's trick 
to convert them to equality checks q(a,b, z) = 0. Thus the degree of our constraints depends upon 
the degree of the polynomials hardwired into the machine. Also, the polynomial constraints use 
real coefficients (involving real numbers hardwired into the machine). Converting these polynomial 
constraints to use only O and 1 as coefficients requires work. The idea is to show that the real 
numbers hardwired into the machine have no effect since the input is a binary string. We omit this 
mathematical argument here. W 


14.3.3 Decidability Questions: Mandelbrot Set 


Since the Blum-Shub-Smale model is more powerful than the ordinary Turing Machine, it makes 
sense to revisit decidability questions. In this section we show that some problems do indeed remain 
undecidable. We study the decidability of the Mandelbrot set with respect to Turing Machines over 
C. Roger Penrose had raised this question in his meditation regarding artificial intelligence. 
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DEFINITION 14.24 (MANDELBROT SET DECISION PROBLEM) 
Let Po(Z) = Z? +C. Then, the Mandelbrot set is defined as 


M = {C | the sequence Po(0), Pe(Pc(0)), Pe(Pe(Pe(0))) ... is bounded }. 


Note that the complement of M is recognizable if we allow inequality constraints. This is 
because the sequence is unbounded iff some number PE(0) has complex magnitude greater than 2 
for some k (exercise!) and this can be detected in finite time. However, detecting that P&(0) is 
bounded for every k seems harder. Indeed, we have: 


THEOREM 14.25 
M is undecidable by a machine over C. 


PROOF: (Sketch) The proof uses the topology of the Mandelbrot set. Let M be any TM over the 
complex numbers that supposedly decides this set. Consider T steps of the computation of this 
TM. Reasoning as in Theorem 14.23 and in our theorems about algebraic computation trees, we 
conclude that the sets of inputs accepted in T steps is a finite union of semialgebraic sets (i.e., sets 
defined using solutions to a system of polynomial equations). Hence the language accepted by M 
is a countable union of semi-algebraic sets, which implies that its Hausdorft dimension is 1. But it 
is known Mandelbrot set has Hausdorff dimension 2, hence M cannot decide it. W 


Exercises 


$1 Show that if field F is finite then arithmetic circuits have exactly the same power —up to 
constant factors—as boolean circuits. 


§2 Equivalence of circuits of depth d to straight line programs of size exp(d). (Lecture 19 in 
Madhu’s notes.) 


$3 Bauer-Strassen lemma? 


84 If function computed in time T on algebraic TM then it has algebraic computation tree of 
depth O(d). 


§5 Prove that if we give the BSS model (over R) the power to test “a > 0?” with arbitrary preci- 
sion, then all of P/poly can be decided in polynomial time. (Hint: the machine’s “program” 
can contain a constant number of arbitrary real numbers.) 


86 Shamir’s trick? 


Chapter notes and history 


NEEDS A LOT 
General reference on algebraic complexity 
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P. Brgisser, M. Clausen, and M. A. Shokrollahi, Algebraic complexity theory, Springer-Verlag, 
1997. 

Best reference on BSS model 

Blum Cucker Shub Smale. 

Algebraic P and NP from Valiant 81 and Skyum-Valiant’86. 

Roger Penrose: emperor’s new mind. 

Mandelbrot : fractals. 
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DRAFT 


Chapter 15 


Average Case Complexity: Levin’s 
Theory 


NEEDS MORE WORK 

Our study of complexity —- NP-completeness, #P-completeness etc.— thus far only concerned 
worst-case complexity. However, algorithms designers have tried to design efficient algorithms for 
NP-hard problems that work for “many” or “most” instances. This motivates a study of the 
difficulty of the “average” instance. Let us first examine the issues at an intuitive level, so we may 
be better prepared for the elements of the theory we will develop. 

Many average case algorithms are targeted at graph problems in random graphs. One can define 
random graphs in many ways: the simplest one generates a graph on n vertices randomly by picking 
each potential edge with probability 1/2. (This method ends up assigning equal probability to every 
n-vertex graph.) On such rand om graphs, many NP-complete problems are easy. 3-COLOR can 
be solved in linear time with high probability (exercise). CLIQUE and INDEPENDENT SET can 
be solved in n?1%87 time (exercise) which is only a little more than polynomial and much less than 
2°", the running time of the best algorithms on worst-case instances. 

However, other NP-complete problems appear to require exponential time even on average. One 
example is SUBSET SUM: we pick n integers a1, a2,...,@, randomly from [1,2”], pick a random 
subset S of {1,...,n}, and produce b = > ¡¿ga¡. We do not know of any efficient average-case 
algorithm that, given the a¿'s and b, finds S. Surprisingly, efficient algorithms do exist if the a;'s are 
picked randomly from the slightly larger interval [1, gn log? "|. This illustrates an important point, 
namely, that average-case complexity is sensitive to the choice of the input distribution. 

The above discussion suggests that even though NP-complete problems are essentially equiva- 
lent with respect to worst case complexity, they may differ vastly in their average case complexity. 
Can we nevertheless identify some problems that remain “complete” even for the average case; in 
other words, are at least as hard as every other average-case NP problem? 

This chapter covers Levin’s theory of average-case complexity. We will formalize the notion 
of “distributional problems,” introduce a working definition of “algorithms that are efficient on 


This chapter written with Luca Trevisan 
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NOTE 15.1 (IMPAGLIAZZO’S POSSIBLE WORLDS) 
At the moment we don’t know if the best algorithm for 3SAT runs in time 


O(n) or 2°") but there are also many other qualitative open questions about 
the hardness of problems in NP. Russell Impagliazzo characterized a central 
goal of complexity theory as the question of finding out which of the following 
possible worlds is the world we live in: 

Algorithmica. 

Heuristica. 


Pessiland. 


Minicrypt. 


average,” and define a reduction that preserves efficient average-case solvability. We will also 
exhibit an NP-complete problem that is complete with respect to such reductions. However, we 
cannot yet prove the completeness of natural distributional problems such as SUBSET SUM or one 
of the number theoretic problems described in the chapter on cryptography. 


15.1 Distributional Problems 


In our intuitive discussion of average case problems, we first fixed an input size n and then considered 
the average running time of the algorithm when inputs of size n are chosen from a distribution. At 
the back of our mind, we knew that complexity has to be measured asymptotically as a function of 
n. To formalize this intuitive discussion, we will define distributions on all (infinitely many) inputs. 


DEFINITION 15.2 (DISTRIBUTIONAL PROBLEM) 
A distributional problem is a pair (L,D), where L is a decision problem and D is a distribution 
over the set [0,1)* of possible inputs. 


EXAMPLE 15.3 
We can define the “uniform distribution” to be one that assigns an input x € {0,1}* the probability 


= 1 
el + [ol) 


We call this “uniform” because it assigns equal probabilities to all strings with the same length. 
It is a valid distribution because the probabilities sum to 1: 


Pr [z] oll, (1) 
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97 n 
== NN a, 2 
5 aa D n(n +1) (2) 


xe€{0,1}* 


Here is another distribution; the probabilities sum to 1 since Sn + = 17/6. 


6 27/2! 


Pre] a 


if |a| >1 (3) 


To pick a string from these distributions, we can first an input length n with the appropriate 
probability (for the distribution in (2), we pick n with probability 6/12n?) and then pick x uniformly 
from inputs of length n. This uniform distribution corresponds to the intuitive approach to average 
case complexity discussed in the introduction. However, the full generality of Definition 15.2 will 
be useful later when we study nonuniform input distributions. 


15.1.1 Formalizations of “real-life distributions.” 


Real-life problem instances arise out of the world around us (images that have to be understood, 
a building that has to be navigated by a robot, etc.), and the world does not spend a lot of 
time tailoring instances to be hard for our algorithm —arguably, the world is indifferent to our 
algorithm. One may formalize this indifference in terms of computational effort, by hypothesizing 
that the instances are produced by an efficient algorithm. We can formalize this in two ways. 


Polynomial time computable distributions. Such distributions have an associated determin- 
istic polynomial time machine that, given input x, can compute the cumulative probability 


up(x), where 
= Y Pri] (4) 
ysr 


Here Prp[y] denotes the probability assigned to string y and y < x means y either precedes x 
in lexicographic order or is equal to x. Denoting the lexicographic predecessor of x by x — 1, 
we have 


Pr[z] = w(x) — u(x — 1), (5) 


which shows that if wp is computable in polynomial time, then so is Prp[x]. The uniform 
distributions in (1) and (1) are polynomial time computable, as are many other distributions 
that are defined using explicit formulae. 


Polynomial time samplable distributions. These distributions have an associated probabilis- 
tic polynomial time machine that can produce samples from the distribution. In other words, 
it outputs x with probability Prp[z]. The expected running time is polynomial in the length 
of the output |æ]. 


Many such samplable distributions are now known, and the sampling algorithm often uses 
Monte Carlo Markov Chain (MCMC) techniques. 
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If a distribution is polynomial time computable then we can efficiently produce samples from 
it. (Exercise.) However, if P 4 P*P there are polynomial time samplable distributions (including 
some very interesting ones) that are not polynomial time computable. (See exercises.) 

In this lecture, we will restrict attention to distributional problems involving a polynomial time 
computable distribution. This may appear to a serious limitation, but with some work the results 
of this chapter can be generalized to samplable distributions. 


15.2 DistNP and its complete problems 


The following complexity class is at the heart of our study of average case complexity. 


dist NP = {(L,D) : L € NP,D polynomial-time computable} . (6) 


Since the same NP language may have different complexity behavior with respect to two different 
input distributions (SUBSET SUM was cited earlier as an example), the definition wisely treats the 
two as distinct computational problems. Note that every problem mentioned in the introduction 
to the chapter is in dist NP. 

Now we need to define the average-case analogue of P. 


15.2.1 Polynomial-Time on Average 


Now we define what it means for a deterministic algorithm A to solve a distributional problem 
(L,D) in polynomial time on average. The definition should be robust to simple changes in model 
of computation or representation. If we migrate the algorithm to a slower machine that has a 
quadratic slowdown (so t steps now take t? time), then polynomial-time algorithms should not 
suddenly turn into exponential-time algorithms. (This migration to a slower machine is not merely 
hypothetical, but also one way to look at a reduction.) As we will see, some intuitively appealing 
definitions do not have this robustness property. 

Denote by t(x) the running time of A on input z. First, note that D is a distribution on all 
possible inputs. The most intuitive choice of saying that A is efficient if 


E/t(x)] is small 


is problematic because the expectation could be infinite even if A runs in worst-case polynomial 
time. 

Next, we could try to define A to be polynomial provided that for some constant c and for every 
sufficiently large n, 


E[e(2)| le] =n] < n° 


This has two problems. First, it ignores the possibility that there could be input lengths on 
which A takes a long time, but that are generated with very low probability under D. In such 
cases A may still be regarded as efficient, but the definition ignores this possibility. Second, and 
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more seriously, the definition is not robust to changes in computational model. To give an example, 
suppose D is the uniform distribution and t(x9) = 2” for just one input zo of size n For every other 
input of size n, t(x) =n. Then Elt(x) | |z| =n] < n+ 1. However, changing to a model with a 
quadratic slowdown will square all running times, and E[(t(x))? | |z| =n] > 2”. 

We could try to define A to be polynomial if there is a c > 0 such that 


E 2] ~ O(1), 


[e] 


but this is also not robust. (Verify this!) 
We now come to a satisfying definition. 


DEFINITION 15.4 (POLYNOMIAL ON AVERAGE AND DIST P) 

A problem (L,D) € dist NP is said to be in dist P if there is an algorithm A for L that satisfies 
for some constants c, c1 

t(x)! 


E 


] =c1, (7) 
where t(x) is the running time of A on input z. 


Notice that P C dist P: if a language can be decided deterministically in time t(x) = O(|z|°), 
then t(x)!/¢ = O(|a|) and the expectation in (7) converges regardless of the distribution. Second, 
the definition is robust to changes in computational models: if the running times get squared, we 
just multiply c by 2 and the expectation in (7) again converges. 

We also point out an additional interesting property of the definition: there is a high probability 
that the algorithm runs in polynomial time. For, if 


t 1/c 
E, = ĉj, (8) 
|x| 
then we have r 
c t(x) s E c 
Peak = Pe 2h] Sa (9) 


where the last claim follows by Markov’s inequality. Thus by increasing k we may reduce this 
probability as much as required. 


15.2.2 Reductions 


Now we define reductions. Realize that we think of instances as being generated according to a 
distribution. Defining a mapping on strings (e.g., a reduction) gives rise to a new distribution on 
strings. The next definition formalizes this observation. 


DEFINITION 15.5 
If f is a function mapping strings to strings and D is a distribution then the distribution f o D is 


one that assigns to string y the probability >>... ey PID [x] 


is proof. 
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DEFINITION 15.6 (REDUCTION) 
A distributional problem (L1, D1) reduces to a distributional problem (L2,D2) (denoted (L1, D1) < 
(La, D2)) if there is a polynomial-time computable function f and an e > 0 such that: 


l. x € Li iff f(x) € Lo. 
2. For every x, |f(x)| = Q(|z|*). 
3. There are constants c, cı such that for every string y, 


Pr < c ly|° Pry). Domination 
Fs, Serle Er@). ) 


The first condition is standard for many-to-one reductions, ensuring that a decision algorithm 
for Lə easily converts into a decision algorithm for Lı. The second condition is a technical one, 
needed later. All interesting reductions we know of satisfy this condition. Next, we motivate the 
third condition, which says that Da “dominates” (up to a polynomial factor) the distribution f oD, 
obtained by applying f on Di. 

Realize that the goal of the definition is to ensure that “if (L1, D1) is hard, then so is (La, D2)” 
(or equivalently, the contrapositive “if (La, D2) is easy, then so is (L1, D1).”) Thus if an algorithm 
Ag is efficient for problem (L2,D2), then the following algorithm ought to be efficient for problem 
(Lı, Dı): on input x obtained from distribution D1, compute f(x) and then run algorithm Ag on 
f(x). A priori, one cannot rule out the possibility that that Ag is very slow on some inputs, which 
are unlikely to be sampled according to distribution D2 but which show up with high probability 
when we sample x according to Dı and then consider f(x). The domination condition helps rule 
out this possibility. 

In fact we have the following result, whose non-trivial proof we omit. 


THEOREM 15.7 
If (Lı, D1) < (L2, D2) and (L2, D2) has an algorithm that is polynomial on average, then (L1, D1) 
also has an algorithm that is polynomial on average. 


Of course, Theorem 15.7 is useful only if we can find reductions between interesting problems. 
Now we show that this is the case: we exhibit a problem (albeit an artificial one) that is complete 
for dist NP. Let the inputs have the form (M al’, 1) where M is an encoding of a Turing 
machine and 1% is a sequence of t ones. Then we define the following “universal” problem U. 


e Decide whether there exists a string y such that |y| < l and M (x,y) accepts in at most t 
steps. 


Since part of the input is in unary, we need to modify our definition of a “uniform” distribution 
to the following. 


l 7 1 1 1 
> (2,1 1) JM |(M +1) jæ (e +1) 21 +EH ao) 
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This distribution is polynomial-time computable (exercise). 


THEOREM 15.8 (LEVIN) 
(U,D) is complete for dist NP, where D is the uniform ditribution. 


The proof requires the following lemma, which shows that for polynomial-time computable dis- 
tributions, we can apply a simple transformation on the inputs such that the resulting distribution 
has no “peaks” (i.e., no input has too high a probability). 


LEMMA 15.9 (PEAK ELIMINATION) 
Suppose D is a polynomial-time computable distribution over x. Then there is a polynomial-time 
computable function g such that 


1. g is injective: g(x) = g (z) if x =z. 
2. |g(x)| < [2] +1. 
3. For every string y, Prgop(y) < 27PI+, 


PROOF: For any string x such that Prp(x) > 27!*!, define h(x) to be the largest common prefix 
of binary representations of p(x), “p(x — 1). Then h is polynomial-time computatable since 
pola) — upla — 1) = Prp(x) > 27!*!, which implies that p(x) and up(£ — 1) must differ in the 
somewhere in the first |x| bits. Thus |h(x)| < log1/ Prp (x) < |x|. Furthermore, h is injective 
because only two binary strings sı and sa can have the longest common prefix z; a third string s3 
sharing z as a prefix must have a longer prefix with either sı or so. 

Now define la 

3 — |t 
f= fe if Prp (2) <2 (11) 

1h(x) otherwise 


Clearly, g is injective and satisfies |g(a)| < |x| + 1. We now show that g oD does not give 
probability more than 27!¥/+! to any string y. If y is not g(x) for any 2, this is trivially true since 
Prgop(y) = 0. 

If y = Ox, where Prp (x) < 27!*!, then Prgop(y) < 2-l¥l+1 and we also have nothing to prove. 

Finally, if y = g(a) = 1h(x) where Prp (x) > 27!*!, then as already noted, |h(a)| < log 1/Prp(x) 
and so Prgop(y) = Prp(x) < 27+, 

Thus the Lemma has been proved. W 


Now we are ready to prove Theorem 15.8. 


PROOF: (Theorem 15.8) At first sight the proof may seem trivial since U is just the “universal” 
decision problem for nondeterministic machines, and every NP language trivially reduces to it. 
However, we also need to worry about the input distributions and enforce the domination condition 
as required by Definition 15.6. 

Let (L, Dı) € dist NP. Let M be a proof-checker for language L that runs in time n°; in 
other words, x € L iff there is a witness y of length |y| = |x|" such that M(a,y) = Accept. (For 
notational ease we drop the big-O notation in this proof.) In order to define a reduction from L to 
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U, the first idea would be to map input x for L to (m, Ü; lal", ier), However, this may violate 


the domination condition because the uniform distribution assigns a probability 2~!*!/poly(|x|) to 
(M, x, piel") whereas x may have much higher probability under Dı. Clearly, this difficulty arises 
only if the distribution Dı has a “peak” at x, so we see an opportunity to use Lemma 15.9, which 
gives us an injective mapping g such that go Dı has no “peaks” and g is computable say in nf time 
for some fixed constant d. 

The reduction is as follows: map x to (M', g(x), 1! +1, plel+lel*y Here M’ is a modification 
of M that expects as input a string z and a witness (x, y) of length |x| + |x|". Given (z, x,y) where 
y = |x|", M’ checks in |x|? time if g(x) = z. If so, it simulates M on (x,y) and outputs its answer. 
If g(x) 4 z then M” rejects. 


To check the domination condition, note that y = (M’, g(x), 1! +lel, plel*+el*) has probability 


Gp -lM 2- lg(2)| 1 

rly) = i ` c c 

D IM’ (M +1) laa (lg) +1) (æ+ 2x + |H] + 2|x|° + |x|? +1) 
ge 1 


.2-9(2) 
< TFD ep a 


under the uniform distribution whereas 


Pr(x) < Q-9(@)+1 <G gee Pr(y) 
Di D 
if we allow the constant G to absorb the term 2!"l|M’| (|M"| +1). Thus the domination condition 
is satisfied. 

Notice, we rely crucially on the fact that 2141 |M'| (M'| + 1) is a constant once we fix the 
language L; of course, this constant will usually be quite large for typical NP languages, and this 
would be a consideration in practice. W 


15.2.3 Proofs using the simpler definitions 


In the setting of one-way functions and in the study of the average-case complexity of the permanent 
and of problems in EXP (with applications to pseudorandomness), we normally interpret “average 
case hardness” in the following way: that an algorithm of limited running time will fail to solve 
the problem on a noticeable fraction of the input. Conversely, we would interpret average-case 
tractability as the existence of an algorithm that solves the problem in polynomial time, except on 
a negligible fraction of inputs. This leads to the following formal definition. 


DEFINITION 15.10 (HEURISTIC POLYNOMIAL TIME) 
We say that an algorithm A is a heuristic polynomial time algorithm for a distributional problem 
(L, u) if A always runs in polynomial time and for every polynomial p 


Y #(@)p(lel) = 001) 


v:A(x)AX1 (x) 
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In other words, a polynomial time algorithm for a distributional problem is a heuristic if the 
algorithm fails on a negligible fraction of inputs, that is, a subset of inputs whose probability 
mass is bounded even if multiplied by a polynomial in the input length. It might also make sense 
to consider a definition in which A is always correct, although it does not necessarily work in 
polynomial time, and that A is heuristic polynomial time if there is a polynomial q such that for 
every polynomial p, ES, u'(x)p(|x|) = O(1), where Sq is the set of inputs x such that A(z) 
takes more than q(|z|) time. Our definition is only more general, because from an algorithm A as 
before one can obtain an algorithm A satisfying Definition 15.10 by adding a clock that stops the 
computation after q(|x|) steps. 

The definition of heuristic polynomial time is incomparable with the definition of average poly- 
nomial time. For example, an algorithm could take time 2” on a fraction 1/n!°8” of the inputs of 
length n, and time n? on the remaining inputs, and thus be a heuristic polynomial time algorithm 
with respect to the uniform distribution, while not beign average polynomial time with respect 
to the uniform distribution. On the other hand, consider an algorithm such that for every input 
length n, and for 1 < k < on? there is a fraction about 1/k? of the inputs of length n on which 
the algorithm takes time O(kn). Then this algorithm satisfies the definition of average polynomial 
time under the uniform distribution, but if we impose a polynomial clock there will be an inverse 
polynomial fraction of inputs of each length on which the algorithm fails, and so the definition of 
heuristic polynomial time cannot be met. 

It is easy to see that heuristic polynomial time is preserved under reductions. 


THEOREM 15.11 
Tf (Li, p1) < (La, u2) and (La, 2) admits a heuristic polynomial time algorithm, then (Li, 11) also 
admits a heuristic polynomial time algorithm. 


PROOF: Let 42 be the algorithm for (La, u2), let f be the function realizing the reduction, and let 
p be the polynomial witnessing the domination property of the reduction. Let c and e be such that 
for every x we have |x| < c|f(x)|!/¢. 

Then we define the algorithm A; than on input x outputs A2(f(1)). Clearly this is a polynomial 
time algorithm, and whenever Ag is correct on f(x), then Ay is correct on z. We need to show that 
for every polynomial q 


`> 1 («)q(a|) = O(1) 
u:A2(f(0) 4x9 (f(x) 
and the left-hand side can be rewritten as 


> XO ui (a)a(lal) 


y: Asly)fxLo (y) 2: f(x)=y 


< E Y doe 
y:A2ly)AxLo (y) z: f(x)=y 
= Y pa(y)p(lyl)a' (yl) 


y: A2ly)fxLo (Y) 
= _O(1) 
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where the last step uses the fact that As is a polynomial heuristic for (La, 12) and in the second- 
to-last step we introduce the polynomial q'(n) defined as q(c- n$) 
A 


15.3 Existence of Complete Problems 


We now show that there exists a problem (albeit an artificial one) complete for dist NP. Let the 
inputs have the form (M 51, 1%), where M is an encoding of a Turing machine and 1° is a sequence 
of t ones. Then we define the following “universal” problem U. 


e Decide whether there exists a string y such that |y| < l and M (x,y) accepts in at most t 
steps. 


That U is NP-complete follows directly from the definition. Recall the definition of NP: we 
say that L € NP if there exists a machine M running in t = poly (|x|) steps such that x € L iff 
there exists a y with y = poly (|x|) such that M (x,y) accepts. Thus, to reduce L to U we need 
only map z onto R (1) = (M, 5,17. 1) where t and / are sufficiently large bounds. 


15.4 Polynomial-Time Samplability 


DEFINITION 15.12 (SAMPLABLE DISTRIBUTIONS) 
We say that a distribution y is polynomial-time samplable if there exists a probabilistic algorithm 
A, taking no input, that outputs x with probability w(x) and runs in poly (|x|) time. 


Any polynomial-time computable distribution is also polynomial-time samplable, provided that 
for all a, 


pa (x) > 27 Poly Uh or p (x) = 0. (13) 


For a polynomial-time computable y satisfying the above property, we can indeed construct a 
sampler A that first chooses a real number r uniformly at random from [0,1], to poly (|x|) bits of 
precision, and then uses binary search to find the first x such that p(x) >r. 

On the other hand, under reasonable assumptions, there are efficiently samplable distributios 
p that are not efficiently computable. 

In addition to dist NP, we can look at the class 


(NP, P-samplable) = { (L, 1) : L € NP, y polynomial-time samplable} . (14) 


A result due to Impagliazzo and Levin states that if (L, u) is dist NP-complete, then (L, yu) is 
also complete for the class (NP, P-samplable). 


15.4. 
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This means that the completeness result established in the previous section extends to the class 
of NP problems with samplable distributions. 


Exercises 


$1 


82 


$3 


$4 


$5 


$6 


Describe an algorithm that decides 3-colorability on almost all graphs in linear expected time. 


"SODIJIDA p UO 
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Describe an algorithm that decides CLIQUE on almost all graphs in n?!°8” time. 
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Show that if a distribution is polynomial-time computable, then it is polynomial-time sam- 
pleable. 


‘yoreos Arewg :JUTH 


Show that if P*P Æ P then there is a polynomial time samplable distribution that is not 
polynomial time computable. 


Show that the function g defined in Lemma 15.9 (Peak Elimination) is efficiently invertible 
in the following sense: if y = g(x), then given y we can reconstruct x in [290 time. 


Show that if one-way functions exist, then dist NP £ dist P. 


Chapter notes and history 


Suppose P 4 NP and yet dist NP C dist P. This would mean that generating hard instances of NP 
problems requires superpolynomial computations. Cryptography is thus impractical. Also, it seems to imply 


that everyday instances of NP-complete problems would also be easily solvable. Such instances arise from 


the world around us —we want to understand an image, or removing the obstacles in the path of a robot— 


and it is hard to imagine how the inanimate world would do the huge amounts of computation necessary to 


generate a hard instance. 
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Chapter 16 


Derandomization, Expanders and 
Extractors 


“God does not play dice with the universe” 
Albert Einstein 


“Anyone who considers arithmetical methods of producing random digits is, of 
course, in a state of sin.” 
John von Neumann, quoted by Knuth 1981 


“How hard could it be to find hay in a haystack?” 
Howard Karloff 


The concept of a randomized algorithm, though widespread, has both a philosophical and a 
practical difficulty associated with it. 

The philosophical difficulty is best represented by Einstein’s famous quote above. Do random 
events (such as the unbiased coin flip assumed in our definition of a randomized turing machine) 
truly exist in the world, or is the world deterministic? The practical difficulty has to do with 
actually generating random bits, assuming they exist. A randomized algorithm running on a 
modern computer could need billions of random bits each second. Even if the world contains some 
randomness —say, the ups and downs of the stock market — it may not have enough randomness to 
provide billions of uncorrelated random bits every second in the tiny space inside a microprocessor. 
Current computing environments rely on shortcuts such as taking a small “fairly random looking” 
bit sequence—e.g., interval between the programmer’s keystrokes measured in microseconds—and 
applying a deterministic generator to turn them into a longer sequence of “sort of random looking” 
bits. Some recent devices try to use quantum phenomena. But for all of them it is unclear how 
random and uncorrelated those bits really are. 


p16.1 (281) 
Complexity Theory: A Modern Approach. (C) 2006 Sanjeev Arora and Boaz Barak. References and attributions are 
still incomplete. 
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Such philosophical and practical difficulties look deterring; the philosophical aspect alone has 
been on the philosophers” table for centuries. The results in the current chapter may be viewed as 
complexity theory’s contribution to these questions. 

The first contribution concerns the place of randomness in our world. We indicated in Chap- 
ter 7 that randomization seems to help us design more efficient algorithms. Á surprising conclusion 
in this chapter is this could be a mirage to some extent. If certain plausible complexity-theoretic 
conjectures are true (e.g., that certain problems can not be solved by subexponential-sized circuits) 
then every probabilistic algorithm can be simulated deterministically with only a polynomial slow- 
down. In other words, randomized algorithms can be derandomized and BPP = P. Nisan and 
Wigderson [NW94] named this research area Hardness versus Randomness since the existence of 
hard problems is shown to imply derandomization. Section 16.3 shows that the converse is also 
true to a certain extent: ability to derandomize implies circuit lowerbounds (thus, hardness) for 
concrete problems. Thus the Hardness + Randomness connection is very real. 

Is such a connection of any use at present, given that we have no idea how to prove circuit 
lowerbounds? Actually, yes. Just as in cryptography, we can use conjectured hard problems in 
the derandomization instead of provable hard problems, and end up with a win-win situation: if 
the conjectured hard problem is truly hard then the derandomization will be successful; and if the 
derandomization fails then it will lead us to an algorithm for the conjectured hard problem. 

The second contribution of complexity theory concerns another practical question: how can we 
run randomized algorithms given only an imperfect source of randomness? We show the existence 
of randomness extractors: efficient algorithms to extract (uncorrelated, unbiased) random bits 
from any weakly random device.Their analysis is unconditional and uses no unproven assumptions. 
Below, we will give a precise definition of the properties that such a weakly random device needs 
to have. We do not resolve the question of whether such weakly random devices exist; this is 
presumably a subject for physics (or philosophy). 

A central result in both areas is Nisan and Wigderson’s beautiful construction of a certain 
pseudorandom generator. This generator is tailor-made for derandomization and has somewhat 
different properties than the secure pseudorandom generators we encountered in Chapter 10. 

Another result in the chapter is a (unconditional) derandomization of randomized logspace 
computations, albeit at the cost of some increase in the space requirement. 


EXAMPLE 16.1 (POLYNOMIAL IDENTITY TESTING) 
One example for an algorithm that we would like to derandomize is the algorithm described in 
Section 7.2.2 for testing if a given polynomial (represented in the form of an arithmetic zero) is 
the identically zero polynomial. If p is an n-variable nonzero polynomial of total degree d over a 
large enough finite field F (|F| > 10d will do) then most of the vectors u € F” will satisfy p(u) 4 0 
(see Lemma A.25. Therefore, checking whether p = 0 can be done by simply choosing a random 
u Er F” and applying p on u. In fact, it is easy to show that there exists a set of m?-vectors 
Weds um such that for every such nonzero polynomial p that can be computed by a size m 
arithmetic circuit, there exists an 7 € [m2] for which p(u’) 4 0. 

This suggests a natural approach for a deterministic algorithm: show a deterministic algorithm 
that for every m € N, runs in poly(m) time and outputs a set u!,... wm” of vectors satisfying the 
above property. This shouldn’t be too difficult— after all the vast majority of the sets of vectors 
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have this property, so hard can it be to find a single one? (Howard Karloff calls this task “finding 
a hay in a haystack”). Surprisingly this turns out to be quite hard: without using complexity 
assumptions, we do not know how to obtain such a set, and in Section 16.3 we will see that in fact 
such an algorithm will imply some nontrivial circuit lowerbounds.! 


16.1 Pseudorandom Generators and Derandomization 


The main tool in derandomization is a pseudorandom generator. This is a twist on the definition of 
a secure pseudorandom generator we gave in Chapter 10, with the difference that here we consider 
nonuniform distinguishers —in other words, circuits— and allow the generator to run in exponential 
time. 


DEFINITION 16.2 (PSEUDORANDOM GENERATORS) 
Let R be a distribution over {0,1}”, S € N and e > 0. We say that R is an 
(S, €)-pseudorandom distribution if for every circuit C of size at most S, 


|Pr[C(R) = 1] — Pr[C(Um) = 1]| < € 


where Um denotes the uniform distribution over (0, 1)”. 


If S : N — Nisa polynomial-time computable monotone function (i.e., S(m) > S(n) 
for m > n)? then a function G : {0,1}* — {0,1}* is called an (S(£)-pseudorandom 
generator (see Figure 16.1) if: 


e For every z € (0,1), |G(z)| = S(£) and G(z) can be computed in time 2 for 
some constant c. We call the input z the seed of the pseudorandom generator. 


e For every £ € N, G(U?) is an (S(£)*, 1/10)-pseudorandom distribution. 


REMARK 16.3 
The choices of the constant 3 and 1/10 in the definition of an S(£)-pseudorandom generator are 
arbitrary and made for convenience. 


The relation between pseudorandom generators and simulating probabilistic algorithm is straight- 
forward: 


"Perhaps it should not be so surprising that “finding a hay in a haystack” is so hard. After all, the hardest open 
problems of complexity— finding explicit functions with high circuit complexity— are of this form, since the vast 
majority of the functions from {0,1}” to {0,1} have exponential circuit complexity. 

2We place these easily satisfiable requirements on the function S to avoid weird cases such as generators whose 
output length is not computable or generators whose output shrinks as the input grows. 
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m >| ™ m >| 
o 


Figure 16.1: A pseudorandom generator G maps a short uniformly chosen seed z Er {0,1}* into a longer output 
G(z) € {0,1} that is indistinguishable from the uniform distribution Um by any small circuit C. 


LEMMA 16.4 

Suppose that there exists an S(£)-pseudorandom generator for some polynomial-time computable 
monotone S : N — N. Then for every polynomial-time computable function £ : N  N, BPTIME(S(“(n))) € 
DTIME(2“)) for some constant c. 


PROOF: A language L is in BPTIME(S(/(n))) if there is an algorithm A that on input x € (0, 1)" 
runs in time cS(¢(n)) for some constant c, and satisfies 


[A(e,r) = L(2)] > E 


Pr 
rerto,1]” 3 


where m < S(£(n)) and we define L(x) = 1 if x € L and L(x) = 0 otherwise. 

The main idea is that if we replace the truly random string r with the string G(z) produced 
by picking a random z € {0, a then an algorithm like A that runs in only S(£) time cannot 
detect this switch most of the time, and so the probability 2/3 in the previous expression does not 
drop below 2/3—0.1. Thus to derandomize A, we do not need to enumerate over all r; it suffices to 
enumerates over all z € (0, 16) and check how many of them make A accept. This derandomized 
algorithm runs in exp(¢(n)) time instead of the trivial 2” time. 

Now we make this formal. Our deterministic algorithm B will on input x € {0,1}”, go over all 
z € {0, re, compute A(x,G(z)) and output the majority answer. Note this takes 20) time. 
We claim that for n sufficiently large, the fraction of z’s such that A(x,G(z)) = L(x) is at least 
3 — 0.1. (This suffices to prove that L € DTIME(2“™) as we can “hardwire” into the algorithm 
the correct answer for finitely many inputs.) 

Suppose this is false and there exists an infinite sequence of x’s for which Pr[A(z,G(z)) = 
L(x) < 2/3 — 0.1. Then we would get a distinguisher for the pseudorandom generator —just use 
the Cook-Levin transformation to construct a circuit that computes the function z > A(z, G(z)), 
where g is hardwired into the circuit. This circuit has size O(S(£(n)))? which is smaller than 
S(L(n))* for sufficiently large n. W 


REMARK 16.5 
The proof shows why it is OK to allow the pseudorandom generator in Definition 16.2 to run in 
time exponential in its seed length. The derandomized algorithm enumerates over all possible seeds 
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of length £, and thus would take exponential time (in £) even if the generator itself were to run in 
less than exponential time. 


Notice, these generators have to fool distinguishers that run for less time than they do. By 
contrast, the definition of secure pseudorandom generators (Definition 10.11 in Chapter 10) re- 
quired the generator to run in polynomial time, and yet have the ability to fool distinguishers that 
have super-polynomial running time. This difference in these definitions stems from the intended 
usage. In the cryptographic setting the generator is used by honest users and the distinguisher is 
the adversary attacking the system — and it is reasonable to assume the attacker can invest more 
computational resources than those needed for normal/honest use of the system. In derandom- 
ization, generator is used by the derandomized algorithm, the ”distinguisher” is the probabilistic 
algorithm that is being derandomized, and it is reasonable to allow the derandomized algorithm 
higher running time than the original probabilistic algorithm. 

Of course, allowing the generator to run in exponential time as in this chapter potentially makes 
it easier to prove their existence compared with secure pseudorandom generators, and this indeed 
appears to be the case. (Note that if we place no upperbounds on the generator's efficiency, we 
could prove the existence of generators unconditionally as shown in Exercise 2, but these do not 
suffice for derandomization.) 


We will construct pseudorandom generators based on complexity assumptions, using quan- 
titatively stronger assumptions to obtain quantitatively stronger pseudorandom generators (i.e., 
S(£)-pseudorandom generators for larger functions S). The strongest (though still reasonable) as- 
sumption will yield a 22%-pseudorandom generator, thus implying that BPP = P. These are 
described in the following easy corollaries of the Lemma that are left as Exercise 1. 


COROLLARY 16.6 
1. If there exists a 2-pseudorandom generator for some constant e > 0 then BPP =P. 


2. If there exists a 2“ -pseudorandom generator for some constant e > 0 then BPP C QuasiP = 
DTIME(2P°¥login)), 


3. If there exists an $(¢)-pseudorandom generator for some super-polynomial function S (i.e., 
S(€) =) then BPP C SUBEXP =NM.oDTIME(2”). 


16.1.1 Hardness and Derandomization 


We construct pseudorandom generators under the assumptions that certain explicit functions are 
hard. In this chapter we use assumptions about average-case hardness, while in the next chapter 
we will be able to construct pseudorandom generators assuming only worst-case hardness. Both 
worst-case and average-case hardness refers to the size of the minimum Boolean circuit computing 
the function: 


p16.6 (286) 16.1. PSEUDORANDOM GENERATORS AND DERANDOMIZATION 


DEFINITION 16.7 (HARDNESS) 

Let f : {0,1}" — {0,1} be a Boolean function. The worst-case hardness of f, 
denoted H,,{f), is a function from N to N that maps every n € N to the largest 
number S such that every Boolean circuit of size at most S fails to compute f on 
some input in {0,1}”. 


The average-case hardness of f, denoted Hu f), is a function from N to N that maps 
every n E N, to the largest number S such that Pre ppo 139 1Clx) = f(x) < 5+ 5 
for every Boolean circuit C on n inputs with size at most S. 


Note that for every function f : (0,1)* — {0,1} and n € N, H,,(f)(n) < Hf) (n) < n2”. 


REMARK 16.8 

This definition of average-case hardness is tailored to the application of derandomization, and in 
particular only deals with the uniform distribution over the inputs. See Chapter 15 for a more 
general treatment of average-case complexity. We will also sometimes apply the notions of worst- 
case and average-case to finite functions from (0, 1)” to {0,1}, where H,,.(f) and Hud f) are defined 
in the natural way. (E.g., if f : {0,1}" — {0,1} then H,,.(f) is the largest number S for which every 
Boolean circuit of size at most S fails to compute f on some input in {0,1}".) 


EXAMPLE 16.9 
Here are some examples of functions and their conjectured or proven hardness: 


1. If f is a random function (i.e., for every x € {0,1}* we choose f(x) using an independent 
unbiased coin) then with high probability, both the worst-case and average-case hardness of 
f are exponential (see Exercise 3). In particular, with probability tending to 1 with n, both 
Hd f) (n) and Hyd f) (nm) exceed 209% We will often use the shorthand HL f), Hud f) > 2°99” 
for such expressions. 


2. If f € BPP then, since BPP C P/poly, both H,,(f) and H,,{f) are bounded by some 
polynomial. 


3. It seems reasonable to believe that 3SAT has exponential worst-case hardness; that is, Hy,.(3SAT) > 
2%") Tt is even more believable that NP É P/poly, which implies that H,,.(3SAT) is super- 
polynomial. The average case complexity of 3SAT is unclear, and in any case dependent upon 
the way we choose to represent formulas as strings. 


4. If we trust the security of current cryptosystems, then we do believe that NP contains func- 
tions that are hard on the average. If g is a one-way permutation that cannot be inverted with 
polynomial probability by polynomial-sized circuits, then by Theorem 10.14, the function f 
that maps the pair x,r € {0,1}” to g~!(x) O r has super-polynomial average-case hardness: 
Huf) > n?. (Where 20 r = 37%, tir; (mod 2).) More generally there is a polynomial 
relationship between the size of the minimal circuit that inverts g (on the average) and the 
average-case hardness of f. 
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The main theorem of this section uses hard-on-the average functions to construct pseudorandom 
generators: 


THEOREM 16.10 (CONSEQUENCES OF NW GENERATOR) 

For every polynomial-time computable monotone S : N — N, if there exists a 
constant c and function f € DTIME(2°) such that Had f) > S(n) then there 
exists a constant e > 0 such that an S(el) -pseudorandom generator exists. In 
particular, the following corollaries hold: 


1. If there exists f € E = DTIME(20™)) and e > 0 such that Had f) > 2° then 
BPP =P. 


2. If there exists f € E = DTIME(20”)) and e > 0 such that Hyd f) > 2” then 
BPP C QuasiP. 


3. If there exists f € E = DTIME(29()) such that Hud f) > n“ then BPP C 
SUBEXP. 


REMARK 16.11 

We can replace E with EXP = DTIME(2Poly(")) in Corollaries 2 and 3 above. Indeed, for every 
f € DTIME(2”), the function g that on input x € {0,1}* outputs the f applies to the first 
lx] Ye bits of x is in DTIME(2”) and satisfies H,(g)(n) > Hv{f)(n'/°). Therefore, if there exists 
f € EXP with Haf) > 2 then there there exists a constant e > 0 and a function g € E 


with Hug) > a and so we can replace E with EXP in Corollary 2. A similar observation 
holds for Corollary 3. Note that EXP contains many classes we believe to have hard problems, 
such as NP,PSPACE,6P and more, which is why we believe it does contain hard-on-the-average 
functions. In the next chapter we will give even stronger evidence to this conjecture, by showing it 
is implied by the assumption that EXP contains hard-in-the-worst-case functions. 


REMARK 16.12 
The original paper of Nisan and Wigderson [NW94] did not prove Theorem 16.10 as stated above. 
It was proven in a sequence of works [?]. Nisan and Wigderson only proved that under the same 


assumptions there exists an S'(£)-pseudorandom generator, where S’(¢) = S (evi log(S (vD) for 


some € > 0. Note that this is still sufficient to derive all three corollaries above. It is this weaker 
version we prove in this book. 


16.2 Proof of Theorem 16.10: Nisan-Wigderson Construction 


How can we use a hard function to construct a pseudorandom generator? 
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16.2.1 Warmup: two toy examples 


For starters, we demonstrate this by considering the “toy example” of a pseudorandom generator 
whose output is only one bit longer than its input. Then we show how to extend by two bits. Of 
course, neither suffices to prove Theorem 16.10 but they do give insight to the connection between 
hardness and randomness. 


Extending the input by one bit using Yao’s Theorem. 
The following Lemma uses a hard function to construct such a “toy” generator: 


LEMMA 16.13 (ONE-BIT GENERATOR) 
Suppose that there exist f € E with H,{f) > nt. Then, there exists an S(¢)-pseudorandom 
generator G for S(¢@) = +1. 


PROOF: The generator G will be very simple: for every z € {0,1}*, we set 
G(s) = zo f(2) 


(where o denotes concatenation). G clearly satisfies the output length and efficiency requirements 
of an (€+1)-pseudorandom generator. To prove that its output is 1/10-pseudorandom we use Yao’s 
Theorem from Chapter 10 showing that pseudorandomness is implied by unpredictiability:? 


THEOREM 16.14 (THEOREM 10.12, RESTATED) 
Let Y be a distribution over (0,1). Suppose that there exist S > 10n,e > 0 such that for every 
circuit C of size at most 2S and i € [m], 


Pr [C(r1, ... Ti) = ral < 


a 
TERY 2 


= 
m 
Then Y is (S, e)-pseudorandom. 


Using Theorem 16.14 it is enough to show that there does not exist a circuit C of size 2(24+ 1)? < 
(* and a number ¿€ [E + 1] such that 
a 1y_1 
A ees) =>] >3+ OF: (1) 
However, for every i < £, the it” bit of G(z) is completely uniform and independent from the first 


i — 1 bits, and hence cannot be predicted with probability larger than 1/2 by a circuit of any size. 
For i = l + 1, Equation (1) becomes, 


1 1 1 1 
P = H > 
eae al > 2° 20(0+1) 7 ote 


which cannot hold under the assumption that Hud f) > nt. E 


3 Although this theorem was stated and proved in Chapter 10 for the case of uniform Turing machines, the proof 
easily extends to the case of circuits. 
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Extending the input by two bits using the averaging principle. 


We now continue to progress in “baby steps” and consider the next natural toy problem: construct- 
ing a pseudorandom generator that extends its input by two bits. This is obtained in the following 
Lemma: 

LEMMA 16.15 (TWO-BIT GENERATOR) 

Suppose that there exists f € E with H,,f) > nt. Then, there exists an ((+2)-pseudorandom 
generator G. 


PROOF: The construction is again very natural: for every z € {0, 1%, we set 


G(z) = 212/20 F2a1,...,2e/2) O 2p/241* "20 f (2epoqay++ +522): 


Again, the efficiency and output length requirements are clearly satisfied. 
To show G(Uz) is 1/10-pseudorandom, we again use Theorem 16.14, and so need to prove that 
there does not exists a circuit C of size 2(¢+ 1)? and i € [l + 2] such that 


1 1 
Pr [C(r1,. F: 9-9) = ri > Sar 


—. 2 
r=G(Ue) 2 20(€+ 2) 7) 


Once again, (2) cannot occur for those indices 7 in which the it” output of G(z) is truly random, 
and so the only two cases we need to consider are i = 4/2 + 1 andi = £+ 2. Equation (2) cannot 
hold for i = £/2+ 1 for the same reason as in Lemma 16.13. For i = £+ 2, Equation (2) becomes: 


Pr [Cre fier) =f > 5+ mary (3) 


rr'ER{0,1}°/? 


This may seem somewhat problematic to analyze since the input to C contains the bit f(r), 
which C could not compute on its own (as f is a hard function). Couldn’t it be that the input 
f(r) helps C in predicting the bit f(r’)? The answer is NO, and the reason is that r’ and r are 
independent. Formally, we use the following principle (see Section A.3.2 in the appendix): 


THE AVERAGING PRINCIPLE: If A is some event depending on two independent random 
variables X,Y, then there exists some z in the range of X such that 


PriA(z, Y) > TAS, Y)] 


e/2 


Applying this principle here, if (3) holds then there exists a string r € {0,1}°~ such that 
1 1 
r C(r, f(r), r) = f(r)| > 2+ =>. 
vc PE EI) 10> 3+ aac 


(Note that this probability is now only over the choice of r’.) If this is the case, we can “hardwire” 
the 0/2+1 bits ro f(r) to the circuit C and obtain a circuit D of size at most (£+ 2)’ +22 < (¢/2)* 
such that i i 

D / = / =. 

Dr’) =f) > 5 +5 


r DAL, AY? 
r'€p{o,1} 4/2 (442) 


contradicting the hardness of f. Mi 
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Beyond two bits: 


A generator that extends the output by two bits is still useless for our goals. We can generalize the 
proof Lemma 16.15 to obtain a generator G that extends the output by k bits setting 


Glases) = 270 f(z")o 270 f(z?)--- 28 o f(2), (4) 


where z’ is the it” block of £/k bits in z. However, no matter how big we set k and no matter how 
hard the function f is, we cannot get a generator that expands its input by a multiplicative factor 
larger than two. Note that to prove Theorem 16.10 we need a generator that, depending on the 
hardness we assume, has output that can be exponentially larger than the input! Clearly, we need 
a new idea. 


16.2.2 The NW Construction 


The new idea is still inspired by the construction of (4), but instead of taking z!,...,z* to be 
independently chosen strings (or equivalently, disjoint pieces of the input z), we take them to be 
partly dependent by using combinatorial designs. Doing this will allow us to take k so large that 
we can drop the actual inputs from the generator's output and use only f(21) o f(z?)---0 f(z*). 
The proof of correctness is similar to the above toy examples and uses Yao’s technique, except the 
fixing of the input bits has to be done more carefully because of dependence among the strings. 
First, some notation. For a string z € {0,1}‘ and subset I C [é], we define zp; to be |/|-length 
string that is the projection of z to the coordinates in I. For example, zyj. is the first 2 bits of z. 


DEFINITION 16.16 (NW GENERATOR) 

If T = {h,...,Im} is a family of subsets of [£] with each |J;| = l and f:{0,1}" > 
{0,1} is any function then the (Z, f)-NW generator (see Figure 16.2) is the function 
Nw2 : {0,1} — {0,1}™ that maps any z € {0,1} to 


NWE(z) = fern) o Flza) 0 Flim) (5) 


Figure 16.2: The NW generator, given a set system Z = ([1,,...,Imy of size n subsets of [£] and a function 
f : {0,1}" — {0,1} maps a string z € [0,1)” to the output f(z11,),...,f(211,,). Note that these sets are not 
necessarily disjoint (although we will see their intersections need to be small). 
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Conditions on the set systems and function. 


We will see that in order for the generator to produce pseudorandom outputs, function f must 
display some hardness, and the family of subsets must come from an efficiently constructible com- 
binatorial design. 


DEFINITION 16.17 (COMBINATORIAL DESIGNS) 
If d,n, € N are numbers with £ > n > d then a family Z = [1,,...,1,,) of subsets of [£] is an 
(£, n, d) -design if |I;| = n for every j and |I; N Ip| < d for every j 4 k. 


The next lemma yields efficient constructions of these designs and is proved later. 


LEMMA 16.18 (CONSTRUCTION OF DESIGNS) 
There is an algorithm A such that on input £,d,n € N where n > d and £ > 10n?/d, runs for 29 
steps and outputs an (2, n, d)-design T containing 2/1 subsets of [4]. 


The next lemma shows that if f is a hard function and Z is a design with sufficiently good 
parameters, than Nw (Up) is indeed a pseudorandom distribution: 


LEMMA 16.19 (PSEUDORANDOMNESS USING THE NW GENERATOR) 
If T is an (¢,n,d)-design with |Z| = 2%% and f : {0,1}" — {0,1} a function satisfying 2% < 
vV Hud f)(n), then the distribution Nw (Up) is a (Hf) (n)/10, 1/10)-pseudorandom distribution. 


PROOF: Let S denote H,,(f)(n). By Yao’s Theorem, we need to prove that for every i € [2/10] 
there does not exist an S/2-sized circuit C such that 


1 


1 


ZU, 
R=NWL(Z) 


For contradiction's sake, assume that (6) holds for some circuit C and some i. Plugging in the 
definition of nw, Equation (6) becomes: 


1 


10 . 24/10 ` (7) 


1 

Pay [C(F(Z11,), o vd (Lita) = HZ11,)] 2 2 T 
Up 

Letting Z, and Zə denote the two independent variables corresponding to the coordinates of Z 


in I; and [4] \ I; respectively, Equation (7) becomes: 


Pr. [C(fi(Z1, Z2), << foil Z2)) = f(A) = : 


A (8) 
Zi Un 2 10-24/10?” 


where for every 7 € [24/ 10], fj applies f to the coordinates of Z¡ corresponding to I; N J; and the 


coordinates of Z2 corresponding to J; \ J;. By the averaging principle, if (8) holds then there exists 
a string 22 € {0, ie such that 


Pr [C(fi(Z1, 22),---, fi-1(Z1, 22)) = $(21)] > l 


a (9) 
Zi¡Un 2 10-.24/10" 
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We may now appear to be in some trouble, since all of f;(Z1, 22) for j < i—1 do depend upon 
Z1, and the fear is that if they together contain enough information about Z: then a circuit could 
potentially predict f;(Z1) after looking at all of them. To prove that this fear is baseless we use the 
fact that the circuit C is small and f is a very hard function. 

Since |; N J;| < d for j A i, the function Zi + f;(Z1, z2) depends at most d coordinates of zı 
and hence can be computed by a d2@-sized circuit. (Recall that z2 is fixed.) Thus if if (8) holds 
then there exists a circuit B of size 2/10 . d24 + 9/2 < S such that 

1 1 1 1 


= > 
zh, P) = fA 2 5 + 0 > 3+ 5 mu 


But this contradicts the fact that H,,(f)(n) = S. Em 


REMARK 16.20 (BLACK-BOX PROOF) 

Lemma 16.19 shows that if NW (Up) is distinguishable from the uniform distribution Uza/1o by 
some circuit D, then there exists a circuit B (of size polynomial in the size of D and in 2%) that 
computes the function f with probability noticeably larger than 1/2. The construction of this 
circuit B actually uses the circuit D as a black-box, invoking it on some chosen inputs. This 
property of the NW generator (and other constructions of pseudorandom generators) turned out 
to be useful in several settings. In particular, Exercise 5 uses it to show that under plausible 
complexity assumptions, the complexity class AM (containing all languages with a constant round 
interactive proof, see Chapter 8) is equal to NP. We will also use this property in the construction 
of randomness extractors based on pseudorandom generators. 


Putting it all together: Proof of Theorem 16.10 from Lemmas 16.18 and 16.19 
As noted in Remark 16.12, we do not prove here Theorem 16.10 as stated but only the weaker state- 
ment, that given f € E and S$: N —N with H,¿f) > S, we can construct an $’(¢)-pseudorandom 
generator, where $’(¢) = S (eV 10g(S(ev?)) for some e > 0. 

For such a function f, we denote our pseudorandom generator by NW‘. Given input z € (0, 1%, 


the generator NW? operates as follows: 


e Set n to be the largest number such that £ > 100n?/log S(n). Set d = log S(n)/10. Since 
S(n) < 2”, we can assume that £ < 300n?/log S(n). 


e Run the algorithm of Lemma 16.18 to obtain an (£, n, d)-design TZ = {,..., Ipa/s}. 


e Output the first S(n)1/% bits of NW (2). 


Clearly, NW! (z) runs in 29% time. Moreover, since 27 < $(n)!/!°, Lemma 16.19 implies 
that the distribution NW (Up) is (S(n)/10, 1/10)-pseudorandom. Since n > Vélog S(n)/300 > 
vL log S CAY 300 (with the last inequality following from the fact that S is monotone), this con- 
cludes the proof of Theorem 16.10. E 


16.3. DERANDOMIZATION REQUIRES CIRCUIT LOWERBOUNDS p16.13 (293) 


Construction of combinatorial designs. 


All that is left to complete the proof is to show the construction of combinatorial designs with the 
required parameters: 

PROOF OF LEMMA 16.18 (CONSTRUCTION OF COMBINATORIAL DESIGNS): On inputs £, d,n with 
l > 10n?/d, our Algorithm A will construct an (£,n, d)-design Z with 24/10 sets using the simple 
greedy strategy: 


Start with Z = Ø and after constructing T = [1,,...,Iny for m < 24/10, search all 
subsets of |£] and add to Z the first n-sized set I satisfying |I N I;| < d for every j € [m]. 
We denote this latter condition by (*). 


Clearly, A runs in poly(m)2* = 2° time and so we only need to prove it never gets stuck. 
In other words, it suffices to show that if 2 = 10n?/d and {lh,..., Im} is a collection of n-sized 
subsets of [¢] for m < 2/10, then there exists an n-sized subset I C [¢] satisfying (*). We do so by 
showing that if we pick J at random by choosing independently every element x € [£] to be in I 
with probability 2n/£ then: 


Pr[l/| > n] > 0.9 (11) 
Pr[[1 M1, > d < 0.5 2410 (vj € [m]) (12) 


Because the expected size of I is 2n, while the expected size of the intersection JM J; is 
2n?/1 < d/5, both (12) and (11) follow from the Chernoff bound. Yet together these two conditions 
imply that with probability at least 0.4, the set J will simultaneously satisfy (*) and have size at 
least n. Since we can always remove elements from J without damaging (*), this completes the 
proof. M 


16.3 Derandomization requires circuit lowerbounds 


We saw in Section 16.2 that if we can prove certain strong circuit lowerbounds, then we can partially 
(or fully) derandomize BPP. Now we prove a result in the reverse direction: derandomizing BPP 
requires proving circuit lowerbounds. Depending upon whether you are an optimist or a pessimist, 
you can view this either as evidence that derandomizing BPP is difficult, or, as a reason to double 
our efforts to derandomize BPP. 

We say that a function is in AlgP /poly if it can be computed by a polynomial size arithmetic 
circuit whose gates are labeled by +, —, x and +, which are operations over some underlying field 
or ring. We let perm denote the problem of computing the permanent of matrices over the integers. 
(The proof can be extended to permanent computations over finite fields of characteristic > 2.) We 
prove the following result. 


THEOREM 16.21 ([?]) 
P = BPP > NEXP ¢ P/poly or perm € AlgP /poly. 
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REMARK 16.22 

It is possible to replace the “poly” in the conclusion perm 4 AlgP /poly with a subexponential 
function by appropriately modifying Lemma 16.25. It is open whether the conclusion NEXP £ 
P /poly can be similarly strengthened. 


In fact, we will prove the following stronger theorem. Recall the Polynomial Identity Testing 
(ZEROP) problem in which the input consists of a polynomial represented by an arithmetic circuit 
computing it (see Section 7.2.2 and Example 16.1), and we have to decide if it is the identically 
zero polynomial. This problem is in coRP C BPP and we will show that if it is in P then the 
conclusions of Theorem 16.21 hold: 


THEOREM 16.23 (DERANDOMIZATION IMPLIES LOWER BOUNDS) 
If ZEROP € P then either NEXP É P/poly or perm ¢ AlgP /poly. 


The proof relies upon many results described earlier in the book.* Recall that MA is the class 
of languages that can be proven by a one round interactive proof between two players Arthur and 
Merlin (see Definition 8.7). Merlin is an all-powerful prover and Arthur is a polynomial-time verifier 
that can flip random coins. That is, given an input x, Merlin first sends Arthur a “proof” y. Then 
Arthur with y in hand flips some coins and decides whether or not to accept x. For this to be an 
MA protocol, Merlin must convince Arthur to accept strings in L with probability one while at the 
same time Arthur must not be fooled into accepting strings not in L except with probability smaller 
than 1/2. We will use the following result regarding MA: 


LEMMA 16.24 ([BFL91],[BFNW93]) 
EXP C P/poly > EXP = MA. 


PROOF: Suppose EXP C P/poly. By the Karp-Lipton theorem (Theorem 6.14), in this case EXP 
collapses to the second level X$ of the polynomial hierarchy. Hence X5 = PH = PSPACE = 
IP = EXP C P/poly. Thus every L € EXP has an interactive proof, and furtheremore, since 
EXP = PSPACE, we can just the use the interactive proof for TQBF, for which the prover is a 
PSPACE machine. Hence the prover can be replaced by a polynomial size circuit family Cn. Now 
we see that the interactive proof can actually be carried out in 2 rounds, with Merlin going first. 
Given an input x of length n, Merlin gives Arthur a polynomial size circuit C, which is supposed to 
be the Cn for L. Then Arthur runs the interactive proof for L, using C as the prover. Note that if 
the input is not in the language, then no prover has a decent chance of convincing the verifier, so 
this is true also for prover described by C. Thus we have described an MA protocol for L implying 
that EXP C MA and hence that EXP = MA. E 


Our next ingredient for the proof of Theorem 16.23 is the following lemma: 


LEMMA 16.25 
If ZEROP € P, and perm € AlgP/poly. Then PP*" C NP. 


‘This is a good example of “third generation” complexity results that use a clever combination of both “classical” 
results from the 60’s and 70’s and newer results from the 1990’s. 
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PROOF: Suppose perm has algebraic circuits of size n°, and that ZEROP has a polynomial-time 
algorithm. Let L be a language that is decided by an n?-time TM M using queries to a perm- 
oracle. We construct an NP machine N for L. 

Suppose x is an input of size n. Clearly, M’s computation on x makes queries to perm of size 
at most m = n’. So N will use nondeterminism as follows: it guesses a sequence of m algebraic 
circuits C1, C2,...,Cm where C; has size if. The hope is that C; solves perm on 7 x i matrices, 
and N will verify this in poly(m) time. The verification starts by verifying C1, which is trivial. 
Inductively, having verified the correctness of C),...,Cz~1, one can verify that C; is correct using 
downward self-reducibility, namely, that for a t x t matrix A, 


t 
perm(A) = y aj perm(A: i), 
i=1 


where Aj; is the (t — 1) x (t— 1) sub-matrix of A obtained by removing the 1st row and ith column 
of A. Thus if circuit C¿-1 is known to be correct, then the correctness of C; can be checked by 
substituting C;(A) for perm(A) and C;-1(A1,) for perm(A1,;): this yields an identity involving 
algebraic circuits with t? inputs which can be verified deterministically in poly(t) time using the 
algorithm for ZEROP. Proceeding this way N verifies the correctness of C1,...,Cm and then 
simulates MP*™ on input x using these circuits. W 


The heart of the proof is the following lemma, which is interesting in its own right: 
LEMMA 16.26 ([?]) 
NEXP C P/poly > NEXP = EXP. 


PROOF: We prove the contrapositive. Suppose that NEXP 4 EXP and let L € NEXP \ EXP. 
Since L € NEXP there exists a constant c > 0 and a relation R such that 


xE Ls aye {0, pa s.t. R(z, y) holds, 
where we can test whether R(x, y) holds in time olal” for some constant c’. 

For every constant d > 0, let Mg be the following machine: on input x € (0,1)” enumerate 
over all possible Boolean circuits C of size n!4 that take n° inputs and have a single output. For 
every such circuit let tt(C) be the 2”°-long string that corresponds to the truth table of the function 
computed by C. If R(x, tt(C)) holds then halt and output 1. If this does not hold for any of the 
circuits then output 0. 

Since Mg runs in time gn , under our assumption that L ¢ EXP, for every d there exists 
an infinite sequence of inputs Xa = {xi}ien on which My(x;) outputs 0 even though z; € L (note 
that if Ma(x) = 1 then x € L). This means that for every string x in the sequence Ag and every y 
such that R(x, y) holds, the string y represents the truth table of a function on n* bits that cannot 
be computed by circuits of size n!¢, where n = |x|. Using the pseudorandom generator based on 
worst-case assumptions (Theorem ??), we can use such a string y to obtain an ¢¢-pseudorandom 


Oldie 


generator. 
Now, if NEXP C P/poly then as noted above NEXP C MA and hence every language in 
NEXP has a proof system where Merlin proves that an n-bit string is in the language by sending 
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a proof which Arthur then verifies using a probabilistic algorithm of at most nf steps. Yet, if n is 
the input length of some string in the sequence Yq and we are given x € X4 with |x| = n, then we 
can replace Arthur by non-deterministic poly(n@)2"° time algorithm that does not toss any coins: 
Arthur will guess a string y such that R(x, y) holds and then use y as a function for a pseudorandom 
generator to verify Merlin’s proof. 

This means that there is a constant c > 0 such that every language in NEXP can be decided on 
infinitely many inputs by a non-deterministic algorithm that runs in poly(2””)-time and uses n bits 
of advice (consisting of the string x € X4). Under the assumption that NEXP C P/poly we can 
replace the poly(2”) running time with a circuit of size n° where c’ is a constant depending only 
on c, and so get that there is a constant c’ such that every language in NEXP can be decided on 
infinitely many inputs by a circuit family of sizen+n. Yet this can be ruled out using elementary 
diagonalization. Mi 


REMARK 16.27 

It might seem that Lemma 16.26 should have an easier proof that goes along the proof that EXP C 
P/poly > EXP = MA, but instead of using the interactive proof for TQBF uses the multi-prover 
interactive proof system for NEXP. However, we do not know how to implement the provers’ 
strategies for this latter system in NEXP. (Intuitively, the problem arises from the fact that a 
NEXP statement may have several certificates, and it is not clear how we can ensure all provers 
use the same one.) 


We now have all the ingredients for the proof of Theorem 16.23. 


PROOF OF THEOREM 16.23: For contradiction’s sake, assume that the following are all true: 


ZEROP € P (13) 
NEXP C P/poly, (14) 
perm € AlgP /poly. (15) 


Statement (14) together with Lemmas 16.24 and 16.26 imply that NEXP = EXP = MA. Now 
recall that MA C PH, and that by Toda’s Theorem (Theorem 9.11) PH C P#?. Recall also that 
by Valiant’s Theorem (Theorem 9.8) perm is ##P-complete. Thus, under our assumptions 


NEXP C prem, (16) 


Since we assume that ZEROP € P, Lemma 16.25 together with statements (15) and (16) implies 
that NEXP C NP, contradicting the Nondeterministic Time Hierarchy Theorem (Theorem 3.3). 
Thus the three statements at the beginning of the proof cannot be simultaneously true. W 


16.4 Explicit construction of expander graphs 


Recall that an expander graph family is a family of graphs (Gn), z such that for some constants 
A and d, for every n € I, the graph Gn has n-vertices, degree d and its second eigenvalue is at 
most A (see Section 7.B). A strongly explicit expander graph family is such a family where there 
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is an algorithm that given n and the index of a vertex v in Gn, outputs the list of v’s neighbors 
in poly(log(n)) time. In this section we show a construction for such a family. Such construction 
have found several applications in complexity theory and other areas of computer science (one such 
application is the randomness efficient error reduction procedure we saw in Chapter 7). 

The main tools in our construction will be several types of graph products. A graph product is 
an operation that takes two graphs G, G” and outputs a graph H. Typically we're interested in the 
relation between properties of the graphs G,G” to the properties of the resulting graph H. In this 
section we will mainly be interested in three parameters: the number of vertices (denoted n), the 
degree (denoted d), and the 2” largest eigenvalue of the normalized adjacency matrix (denoted A), 
and study how different products affect these parameters. We then use these products to obtain a 
construction of a strongly explicit expander graph family. In the next section we will use the same 
products to show a deterministic logspace algorithm for undirected connectivity. 


16.4.1 Rotation maps. 


In addition to the adjacency matrix representation, we can also represent an n-vertex degree-d 
graph G as a function G from [|n] x [d] to [n] that given a pair (v,i) outputs u where the i*” 
neighbor of v in G. In fact, it will be convenient for us to have G output an additional value j € [d] 
where j is the index of v as a neighbor of u. Given this definition of G it is clear that we can invert 
it by applying it again, and so it is a permutation on [n] x [d]. We call G the rotation map of G. 
For starters, one may think of the case that G(u, i) = (v,i) (i.e., v is the i” neighbor of u iff u is 
the i” neighbor of v). In this case we can think of G as operating only on the vertex. However, 
we will need the more general notion of a rotation map later on. 

We can describe a graph product in the language of graphs, adjacency matrices, or rotation 
maps. Whenever you see the description of a product in one of this forms (e.g., as a way to map 
two graphs into one), it is a useful exercise to work out the equivalent descriptions in the other 
forms (e.g., in terms of adjacency matrices and rotation maps). 


16.4.2 The matrix/path product 


G: (n,d,A)-graph G’: (n,d',1')-graph G'G: (n,dd',22')-graph 


IJ] 


For every two n vertex graphs G, G’ with degrees d,d’ and adjacency matrices A, A’, the graph 
G'G is the graph described by the adjacency matrix A'A. That is, G'G has an edge (u, v) for every 
length 2-path from u to v where the first step in the path is taken on en edge of G and the second 
is on an edge of G’. Note that G has n vertices and degree dd’. Typically, we are interested in 
the case G = G”, where it is called graph squaring. More generally, we denote by G* the graph 
G-G---G (k times). We already encountered this case before in Lemma 7.27, and similar analysis 
yields the following lemma (whose proof we leave as exercise): 
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LEMMA 16.28 (MATRIX PRODUCT IMPROVES EXPANSION) 
A(G'G) < MGA(G”) 


It is also not hard to compute the rotation map of G'G from the rotation maps of G and G’. 
Again, we leave verifying this to the reader. 


16.4.3 The tensor product 


G: (n,d,A)-graph G’: (n’,d’,’)-graph GOG’: (nn’,dd’,max{A.,4’})-graph 


T 
TO 
y 


Ya 


j 


Se 


Let G and G” be two graphs with n (resp n’) vertices and d (resp. d’) degree, and let G : 
În] x [d] => [n] x [d] and G’ : [n] x [d'] > [n'] x [d'] denote their respective rotation maps. The 
tensor product of G and G”, denoted G ® G”, is the graph over nn’ vertices and degree dd’ whose 
rotation map G ® G' is the permutation over ([n] x [n']) x ([d] x [d']) defined as follows 


G a G'((u, v), (i, j)) T (ul, v’), Ga) > 


where (u’, i’) = G(u,i) and (v”, j) = G'(v, j). That is, the vertex set of G & G is pairs of vertices, 
one from G and the other from G”, and taking a the step (i, j) on G & G’ from the vertex (u,v) is 
akin to taking two independent steps: move to the pair (u’,v’) where u’ is the i” neighbor of u in 
G and v’ is the i” neighbor of v in G". 

In terms of adjacency matrices, the tensor product is also quite easy to describe. If A = (aij) 
is the n x n adjacency matrix of G and A’ = (ay jr) is the n’ x n’ adjacency matrix of G”, then the 
adjacency matrix of G@ G", denoted as 48 A’, will be an nn’ x nn’ matrix that in the (i,7’)"" row 
and the (j, 7") column has the value aj,j + ay jr That is, A8 A’ consists of n? copies of A’, with the 


(5,5) copy scaled by aj,;: 
ai A’ a1 94 is a1 nA’ 
A 2 Al a21A’ a22A' eat 02 A 
Oni A an2A' ... annA 


The tensor product can also be described in the language of graphs as having a cluster of n’ 
vertices in G & G” for every vertex of G. Now if, u and v are two neighboring vertices in G, we will 
put a bipartite version of G’ between the cluster corresponding to u and the cluster corresponding 
to v in G. That is, if (i,j) is an edge in G” then there is an edge between the it” vertex in the 
cluster corresponding to u and the jt? vertex in the cluster corresponding to v. 
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LEMMA 16.29 (TENSOR PRODUCT PRESERVES EXPANSION) 
Let \ = A(G) and X = A(G") then A(G & G’) < max{), AY. 


One intuition for this bound is the following: taking a T step random walk on the graph Ge G” 
is akin to taking two independent random walks on the graphs G and G”. Hence, if both walks 
converge to the uniform distribution within T steps, then so will the walk on G 9 G”. 
PROOF: Given some basic facts about tensor products and eigenvalues this is immediate since if 
A1, +++, An are the eigenvalues of A (where A is the adjacency matrix of G) and Aj,..., /,, are the 
eigenvalues of A (where A’ is the adjacency matrix of G”), then the eigenvalues of A ® A’ are all 
numbers of the form A; + Aj, and hence the largest ones apart from 1 are of the form 1 - A(G”) or 
A(G) - 1 (see also Exercise 14). Mi 

We note that one can show that A(G @ G’) < AG) + A(G") without relying on any knowledge 


of eigenvalues (see Exercise 15). This weaker bound suffices for our applications. 


16.4.4 The replacement product 


G: (n,D,1-e)-graph  G”: (D,d,1-e')-graph G@G': (nD,2d,1-ee'/4)-graph 


In both the matric and tensor products, the degree of the resulting graph is larger than the 
degree of the input graphs. The following product will enable us to reduce the degree of one of the 
graphs. Let G,G” be two graphs such that G has n vertices and degree D, and G” has D vertices 
and degree d. The balanced replacement product (below we use simply replacement product for 
short) of G and G” is denoted by G@)G’ is the nn'-vertex 2d-degree graph obtained as follows: 


1. For every vertex u of G, the graph G@G" has a copy of G’ (including both edges and vertices). 


2. If u,v are two neighboring vertices in G then we place d parallel edges between the it” vertex 
in the copy of G” corresponding to u and the jt” vertex in the copy of G’ corresponding to v, 
where i is the index of v as a neighbor of u and j is the index of u as a neighbor of v in G. 
(That is, taking the it edge out of u leads to v and taking the j} edge out of v leads to u.) 


Note that we essentially already encountered this product in the proof of Claim ?? (see also 
Figure ??), where we reduced the degree of an arbitrary graph by taking its replacement product 
with a cycle (although there we did not use parallel edges).? The replacement product also has 


The addition of parallel edges ensures that a random step from a vertex v in GQ) G” will move to a neighbor 
within the same cluster and a neighbor outside the cluster with the same probability. For this reason, we call this 
product the balanced replacement product. 


p16.20 (300) 16.4. EXPLICIT CONSTRUCTION OF EXPANDER GRAPHS 


a simple description in terms of rotation maps: since G HG” has nD vertices and 2d degree, its 
rotation map G@)G’ is a permutation over ([n] x [D]) x ([d] x {0,1}) and so can be thought of as 
taking four inputs u,v,i,b where u € [n], v € [D], i € [d] and b € {0,1}. If b = 0 then it outputs 
u, G'(v, i),b and if b = 1 then it outputs G(u, v), i,b. That is, depending on whether b is equal to 0 
or 1, the rotation map either treats v as a vertex of G’ or as an edge label of G. 

In the language of adjacency matrices the replacement product can be easily seen to be described 
as follows: A@A’ = /2(A9 Ip) +1/2(In @ A’), where A, A’ are the adjacency matrices of the graphs 
G and G” respectively, and I; is the k x k identity matrix. 

If D > d then the replacement product’s degree will be significantly smaller than G’s degree. 
The following Lemma shows that this dramatic degree reduction does not cause too much of a 
deterioration in the graph’s expansion: 


LEMMA 16.30 (EXPANSION OF REPLACEMENT PRODUCT) 
If A(G) < 1 — e and AG”) < 1 — e then A(G® G’) < 1 — ee /4. 


The intuition behind Lemma 16.30 is the following: Think of the input graph G as a good 

expander whose only drawback is that it has a too high degree D. This means that a k step random 
walk on G” requires O(klog D) random bits. However, as we saw in Section 7.B.3, sometimes we 
can use fewer random bits if we use an expander. So a natural idea is to generate the edge labels for 
the walk by taking a walk using a smaller expander G” that has D vertices and degree d < D. The 
definition of G@®G?’ is motivated by this intuition: a random walk on G@®)G’ is roughly equivalent 
to using an expander walk on G” to generate labels for a walk on G. In particular, each step a 
walk over G@®G’ can be thought of as tossing a coin and then, based on its outcome, either taking 
a a random step on G”, or using the current vertex of G” as an edge label to take a step on G. 
Another way to gain intuition on the replacement product is to solve Exercise 16, that analyzes 
the combinatorial (edge) expansion of the resulting graph as a function of the edge expansion of 
the input graphs. 
PROOF OF LEMMA 16.30: Let A (resp. A’) denote the n x n (resp. D x D) adjacency matrix of 
G (resp. G’) and let \(A) = 1 — e and X\(A’) = 1 — e’. Then by Lemma 7.40, A = (1 — €)C + Jn 
and A’ = (1—€')C'+ Jp, where Jẹ is the k x k matrix with all entries equal to 1/k. 

The adjacency matrix of G @) G" is equal to 


(48 Ip) + i(n ® 4) = 1C @ In + $h IN + 51,9C'+51,0 Ip, 


where I; is the k x k identity matrix. 
Thus, the adjacency matrix of (G ®@ G")? is equal to 


1 f 2 
(508 p+ §n@Ip+43{h@C'+$m@Jp) = 


where F is some nD x nD matrix of norm at most 1 (obtained by collecting together all the other 
terms in the expression). But 
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(This can be verified by either direct calculation or by going through the graphical representation 
or the rotation map representation of the tensor and matrix products.) 
Since every vector v € R”? that is orthogonal to 1 satisfies J, py v = 0, we get that 


(COS) =A (COG) =AU1- H)F + FInv) <1- E, 


and hence 


1 


MEE) ZISE 
E 


16.4.5 The actual construction. 


We now use the three graph products of described above to show a strongly explicit construction 
of an expander graph family. Recall This is an infinite family {G;,} of graphs (with efficient way to 
compute neighbors) that has a constant degree and an expansion parameter A. The construction 
is recursive: we start by a finite size graph G (which we can find using brute force search), and 
construct the graph Gk from the graph G,_;. On a high level the construction is as follows: each 
of the three product will serve a different purpose in the construction. The Tensor product allows 
us to take G,_, and increase its number of vertices, at the expense of increasing the degree and 
possibly some deterioration in the expansion. The replacement product allows us to dramatically 
reduce the degree at the expense of additional deterioration in the expansion. Finally, we use the 
Matriz/Path product to regain the loss in the expansion at the expense of a mild increase in the 
degree. 


THEOREM 16.31 (EXPLICIT CONSTRUCTION OF EXPANDERS) 
There exists a strongly-explicit A, d-expander family for some constants d and À < 1. 


PROOF: Our expander family will be the following family {Gk }ken of graphs: 


e Let H be a (D = d*, d,0.01)-graph, which we can find using brute force search. (We choose 
d to be a large enough constant that such a graph exists) 


e Let Gi be a (D,d?, 1/2)-graph, which we can find using brute force search. 
e For k > 1, let Gk = (Gr-1 S Gp-1) @ H)*°, 
The proof follows by noting the following points: 


1. For every k, Gk has at least 22" vertices. 
Indeed, if nz denotes the number of vertices of Gz, then ng = (ng-1)? D. If ng- > 2” then 
22 
wS (2 ‘) = 9% 
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2. For every k, the degree of Gg is d”. 


Indeed, taking a replacement produce with H reduces the degree to d, which is then increased 
to d” by taking the 20" power of the graph (using the matrix/path product). 


3. There is a 20()-time algorithm that given a label of a vertex u in Gy and an index i € [d°], 
outputs the i” neighbor of u in Gz. (Note that this is polylogarithmic in the number of 
vertices. ) 


Indeed, such a recursive algorithm can be directly obtained from the definition of Gz. To 
compute G;’s neighborhood function, the algorithm will make 40 recursive calls to Gz_1’s 
neighborhood function, resulting in 22% running time. 


4. For every k, A(Gk) < 1/3. 


Indeed, by Lemmas 16.28, 16.29, and 16.30 If A(Gx_1) < 1/3 then MGx-_1 Y Gr-1) < 2/3 and 
hence A((G;-19G;-1J0H) < 1-%% < 1-1/13. Thus, A(G) < (1-1/13) ~ e7 70/18 < 1/3, 


Using graph powering we can obtain such a construction for every constant A € (0,1), at the 
expense of a larger degree. There is a variant of the above construction supplying a denser family 
of graphs that contains an n-vertex graph for every n that is a power of c, for some constant c. 
Since one can transform an (n, d, A)-graph to an (n*, cd”, A)-graph for any n/c < n’! < n by making 
a single “mega-vertex” out of a set of at most c vertices, the following theorem is also known: 


THEOREM 16.32 
There exist constants d € N , A < 1 and a strongly-explicit graph family [Gn nen such that Gn is 
an (n, d, A)-graph for every n € N. 


REMARK 16.33 

As mentioned above, there are known constructions of expanders (typically based on number theory) 
that are more efficient in terms of computation time and relation between degree and the parameter 
A than the product-based construction above. However, the proofs for these constructions are more 
complicated and require deeper mathematical tools. Also, the replacement product (and its close 
cousin, the zig-zag product) have found applications beyond the constructions of expander graphs. 
One such application is the deterministic logspace algorithm for undirected connectivity described 
in the next section. Another application is a construction of combinatorial expanders with greater 
expansion that what is implied by the parameter A. (Note that even for for the impossible to 
achieve value of A = 0, Theorem ?? implies combinatorial expansion only 1/2 while it can be shown 
that a random graph has combinatorial expansion close to 1.) 


16.5 Deterministic logspace algorithm for undirected connectiv- 
ity. 


This section describes a recent result of Reingold, showing that at least the most famous random- 
ized logspace algorithm, the random walk algorithm for s-t-connectivity in undirected graphs ( 
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Chapter 7) can be completely “derandomized.” Thus the s-t-connectivity problem in undirected 
graphs is in L. 


THEOREM 16.34 (REINGOLD’S THEOREM |[?]) 
UPATH € L. 


Reingold describes a set of poly(n) walks starting from s such that if s is connected to t then 
one of the walks is guaranteed to hit t. Of course, the existence of such a small set of walks is 
trivial; this arose in our discussion of universal traversal sequences of Definition ??. The point is 
that Reingold's enumeration of walks can be carried out deterministically in logspace. 

In this section, all graphs will be multigraphs, of the form G = (V, E) where E is a multiset 
(i.e., some edges may appear multiple times, and each appearance is counted separately). We say 
the graph is d-regular if for each vertex i, the number of edges incident to it is exactly d. We 
will assume that the input graph for the s-t connectivity problem is d-regular for say d = 4. This 
is without loss of generality: if a vertex has degree d” < 3 we add a self-loop of multiplicity to 
bring the degree up to d, and if the vertex has degree d' > 3 we can replace it by a cycle of d’ 
vertices, and each of the d' edges that were incident to the old vertex then attach to one of the cycle 
nodes. Of course, the logspace machine does not have space to store the modified graph, but it can 
pretend that these modifications have taken place, since it can perform them on the fly whenever 
it accesses the graph. (Formally speaking, the transformation is implicitly computable in logspace; 
see Claim ??.) In fact, the proof below will perform a series of other local modifications on the 
graph, each with the property that the logspace algorithm can perform them on the fly. 

Recall that checking connectivity in expander graphs is easy. Specifically, if every connected 
component in G is an expander, then there is a number / = O(logn) such that if s and t are 
connected then they are connected with a path of length at most £. 


THEOREM 16.35 
If an n-vertex graph G is d-regular graph and \(G) < 1/4 then the maximum distance between 
every pair of nodes is at most O(dlogn). 


PROOF: The exercises ask you to prove that for each subset S of size at most |V| /2, the number 
of edges between S and S is at least (1 — à) |S| /2 > 3|S|/8. Thus at least 3|S| /8d vertices in 
S must be neighbors of vertices in S. Iterating this argument l times we conclude the following 
about the number of vertices whose distance to S is at most l: it is either more than |V| /2 (when 
the abovementioned fact stops applying) or at least (1 + Sy) Let s,t be any two vertices. Using 
S = {s}, we see that at least |V| /2+1 vertices must be within distance l = 10dlog n of s. The same 
is true for vertex t. Every two subsets of vertices of size at least |V| /2 + 1 necessarily intersect, so 
there must be some vertex within distance l of both s and t. Hence the distance from s to t is at 
most 2/. W 


We can enumerate over all /-step random walks of a d-degree graph in O(d¢) space by enu- 
merating over all sequences of indices 71,...,i¢ € [d]. Thus, in a constant-degree graph where all 
connected components are expanders we can check connectivity in logarithmic space. 
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The idea behind Reingold’s algorithm is to transform the graph G (in an implicitly computable 
in logspace way) to a graph G” such that every connected component in G becomes an expander 
in G’, but two vertices that were not connected will stay unconnected. 

By adding more self-loops we may assume that the graph is of degree d?° for some constant d 
that is sufficiently large so that there exists a (42, d,0.01)-graph H. (See Fact ?? in the Appendix.) 
Since the size of H is some constant, we assume the algorithm has access to it (either H could be 
”hardwired” into the algorithm or the algorithm could perform brute force search to discover it). 
Consider the following sequence of transformations. 


e Let Go =G. 


e For k > 1, we define Gy = (Gr-1 @ H)”. 


Here (YH is the replacement product of the graph, defined in Chapter ??. If G—1 is a graph with 
degree d2, then Gy_1 ®H is a graph with degree d and thus Gy = (Gk-1 ®Q H)” is again a graph 
with degree d2 (and size (2d?° |G,_1|)?°). Note also that if two vertices were connected (resp., 
disconnected) in Gk—1, then they are still connected (resp., disconnected) in Gk. Thus to solve the 
UPATH in G it suffices to solve a UPATH problem in any of the G;’s. 

Now we show that for k = O(log n), the graph Gg is an expander, and therefore an easy instance 
of UPATH. By Lemmas 16.28 and 16.30, for every e < 1/20 and D-degree graph F, if MF) <1-—e 
then A(F ® H) < 1—e/5 and hence à ((F®H)”) < 1 — 2e. By Lemma 7.28, every connected 
component of G has expansion parameter at most 1 — 1/(8Dn?), where n denotes the number of 
G’s vertices which is at least as large as the number of vertices in the connect component. It follows 
that for k = 10log Dlog N, in the graph Gk every connected component has expansion parameter 
at most max{1 — 1/20, 2*/(8Dn*)} = 1 — 1/20. 

To finish, we show how to solve the UPATH problem for Gx in logarithmic space for this value of 
k. The catch is of course that the graph we are given is G, not Gk. Given G, we wish to enumerate 
length £ starting from a given vertex in Gk since the graph is an expander. A walk describes, for 
each step, which of the d? outgoing edges to take from the current vertex. Thus it suffices to show 
how we can compute in O(k + logn) space, the ith outgoing edge of a given vertex u in Gy. This 
map’s input length is O(k + logn) and hence we can assume it is placed on a read/write tape, and 
will compute the rotation map “in-place” changing the input to the output. Let są be the additional 
space (beyond the input) required to compute the rotation map of Gy. Note that sy = O(logn). 
We show a recursive algorithm to compute Gę satisfying the equation sk = s,_; + O(1). In fact, 
the algorithm will be a pretty straightforward implementation of the definitions of the replacement 
and matrix products. 

The input to Gx is a vertex in (Gk-1 ® H) and 20 labels of edges in this graph. If we can 
compute the rotation map of Gk-1®)H in s,_; + O(1) space then we can do so for Gp, since we can 
simply make 20 consecutive calls to this procedure, each time reusing the space.? Now, to compute 
the rotation map of (G,_1 (6) H) we simply follow the definition of the replacement product. Given 


“One has to be slightly careful while making recursive calls, since we don’t want to lose even the O(log log n) bits 
of writing down k and keeping an index to the location in the input we're working on. However, this can be done 
by keeping k in global read/write storage and since storing the identity of the current step among the 50 calls we're 
making only requires O(1) space. 
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an input of the form u,v,i,b (which we think of as read/write variables), if b = 0 then we apply 
the rotation map of H to (v,i) (can be done in constant space), while if b = 1 then we apply the 
rotation map of Gz-1 to (u,v) using a recursive call at the cost of sz; space (note that u,v are 
conveniently located consecutively at the beginning of the input tape). 


16.6 Weak Random Sources and Extractors 


Suppose, that despite the philosophical difficulties, we are happy with probabilistic algorithms, and 
see no need to “derandomize” them, especially at the expense of some unproven assumptions. We 
still need to tackle the fact that real world sources of randomness and unpredictability rarely, if 
ever, behave as a sequence of perfectly uncorrelated and unbiased coin tosses. Can we still execute 
probabilistic algorithms using real-world “weakly random” sources? 


16.6.1 Min Entropy 
For starters, we need to define what we mean by a weakly random source. 


DEFINITION 16.36 
Let X be a random variable. The min entropy of X, denoted by H,,(X), is the largest real number 
k such that Pr[X = z] < 27% for every x in the range of X. 

If X is a distribution over {0,1}" with H.(X) > k then it is called an (n, k)-source. 


It is not hard to see that if X is a random variable over (0, 1)” then Hxo(X) < n with H(X) = 
n if and only if X is distributed according to the uniform distribution Un. Our goal in this section is 
to be able to execute probabilistic algorithms given access to a distribution X with H,,(X) as small 
as possible. It can be shown that min entropy is a minimal requirement in the sense that in general, 


to execute a probabilistic algorithm that uses k random bits we need access to a distribution X 
with H..(X) > k (see Exercise ??). 


EXAMPLE 16.37 
Here are some examples for distributions X over (0,1)” and their min-entropy: 


e (Bit fixing and generalized bit fixing sources) If there is subset S C [n] with |S] = k such that 
X’s projection to the coordinates in S is uniform over {0,1}”, and X’s projection to [n] \ S is 
a fixed string (say the all-zeros string) then H,.(X) = k. The same holds if X’s projection to 
[n] \ S is a fixed deterministic function of its projection to S. For example, if the bits in the 
odd positions of X are independent and uniform and for every even position 27, Xo; = Xəi—1 
then H..(X) =| 5 |. This may model a scenario where we measure some real world data at 
too high a rate (think of measuring every second a physical event that changes only every 
minute). 


e (Linear subspaces) If X is the uniform distribution over a linear subspace of GF(2)” of 
dimension k, then Ho(X) = k. (In this case X is actually a generalized bit-fixing source— 
can you see why?) 


p16.26 (306) 16.6. WEAK RANDOM SOURCES AND EXTRACTORS 


e (Biased coins) If X is composed of n independent coins, each outputting 1 with probability 
ô < 1/2 and 0 with probability 1 — ô, then as n grows, H.(X) tends to H(9)n where H is the 
Shannon entropy function. That is, H(d) = 0 log i + (1 — ô) log 3: 


e (Santha-Vazirani sources) If X has the property that for every i € [n], and every string 
x € {0, i: conditioned on Xı = 24,...,Xi-1 = 2-1 it holds that both Pr[X; = 0] and 
Pr|X; = 1] are between ô and 1 — 6 then H,.(X) > H(d)n. This can model sources such as 
stock market fluctuations, where current measurements do have some limited dependence on 
the previous history. 


e (Uniform over subset) If X is the uniform distribution over a set S C (0,1)” with |S| = 2* 
then H(X) = k. As we will see, this is a very general case that “essentially captures” all 
distributions X with H,.(X) = k. 


We see that min entropy is a pretty general notion, and distributions with significant min 
entropy can model many real-world sources of randomness. 


16.6.2 Statistical distance and Extractors 


Now we try to formalize what it means to extract random —more precisely, almost random— bits 
from an (n, k) source. To do so we will need the following way of quantifying when two distributions 
are close. 


DEFINITION 16.38 (STATISTICAL DISTANCE) 

For two random variables X and Y with range {0,1}'", their statistical distance (also known as 
variation distance) is defined as 6(X,Y) = maxgcyo1y~{|Pr[X € S] — Pr[Y e S]|}. We say that 
X,Y are e-close, denoted X œ Y, if 0(X, Y) < e. 


Statistical distance lies in [0,1] and satisfies triangle inequality, as suggested by its name. The 
next lemma gives some other useful properties; the proof is left as an exercise. 


LEMMA 16.39 
Let X,Y be any two distributions taking values in {0,1}”. 
1. 6(X,Y) = 3 emar Pr = 2] — Pr[Y =a]. 


2. (Restatement of Definition 16.38) 6(X,Y) > e iff there is a boolean function D: {0,1} = 
{0,1} such that |Przex[D(x) = 1] — Pryey[D(y) = 1]| > e. 


3. If f : {0,1}" — {0,1}° is any function, then 6(f(X), f(Y)) < 6(X,Y). (Here f(X) is a 
distribution on (0,1)* obtained by taking a sample of X and applying f.) 


Now we define an extractor. This is a (deterministic) function that transforms an (n, k) source 
into an almost uniform distribution. It uses a small number of additional truly random bits, denoted 
by ¢ in the definition below. 
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DEFINITION 16.40 

A function Ext : {0,1}" x {0,1}' > {0,1}” is a (k, e) extractor if for any (n,k)-source X, the 
distribution Ext(X,U;) is e-close to Um. (For every 4, Up denotes the uniform distribution over 


(0,17%) 


Equivalently, if Ext : (0,1)” x {0,1}’ — (0,1)” is a (k, e) extractor, then for every distribution 
X ranging over {0,1}" of min-entropy k, and for every S C {0,1}'", we have 


[Prae x ze{0,1 4 [Ext(a, z) € S] — Pref0,1y" [re S]| <e 


We use this fact to show in Section 16.7.2 how to use extractors and (n, k)-sources to to simulate 
any probabilistic computation. 


Why an additional input? Our stated motivation for extractors is to execute probabilistic 
algorithms without access to perfect unbiased coins. Yet, it seems that an extractor is not sufficient 
for this task, as we only guarantee that its output is close to uniform if it is given an additional 
input that is uniformly distributed. First, we note that the requirement of an additional input is 
necessary: for every function Ext : {0,1}" — {0,1} and every k < n — 1 there exists an (n, k)- 
source X such that the first bit of Ext(X) is constant (i.e, is equal to some value b € {0,1} with 
probability 1), and so is at least of statistical distance 1/2 from the uniform distribution (Exercise 7). 
Second, if the length t of the second input is sufficiently short (e.g., t = O(logn)) then, for the 
purposes of simulating probabilistic algorithms, we can do without any access to true random coins, 
by enumerating over all the 2° possible inputs (see Section 16.7.2). Clearly, t has to be somewhat 
short for the extractor to be non-trivial: for t > m, we can have a trivial extractor that ignores its 
first input and outputs the second input. This second input is called the seed of the extractor. 


16.6.3 Extractors based upon hash functions 


One can use pairwise independent (and even weaker notions of) hash functions to obtain extractors. 
In this section, H denotes a family of hash functions h: {0,1}" —> {0,1}". We say it has collision 
error 6 if for any 11 Æ x2 € {0,1}", Prrem[h(x1) = h(x2)) < (1+ 6)/2*. We assume that one 
can choose a random function h € H by picking a string at random from {0, i We define the 
extractor Ext: x {0,1} > {0,1}*** as follows: 


Ext(x, h) = h(x) on, (17) 


where o denotes concatenation of strings. 
To prove that this is an extractor, we relate the min-entropy to the collision probability of a 
distribution, which is defined as $., p?, where pa is the probability assigned to string a. 


LEMMA 16.41 
If a distribution X has min-entropy at least k then its collision probability is at most 1/2". 


PROOF: For every a in X’s range, let pa be the probability that X = a. Then, YP. < 
maxa {Pa} (24 Pa) < > A >. a 
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LEMMA 16.42 (LEFTOVER HASH LEMMA) 
If x is chosen from a distribution on {0,1}" with min-entropy at least k/5 and H has collision error 
ô, then h(X) oh has distance at most 26 to the uniform distribution. 


PROOF: Left as exercise. (Hint: use the relation between the Lz and Lı norms M 


16.6.4 Extractors based upon random walks on expanders 


This section assumes knowledge of random walks on expanders, as described in Chapter ??. 


LEMMA 16.43 
Let e > 0. For every n and k < n there exists a (k, e)-extractor Ext : {0,1}" x {0,1}! = (0, 13” 
where t = O(n — k + log1/e). 


PROOF: Suppose X is an (n, k)-source and we are given a sample a from it. Let G be a (2”, d, 1/2)- 
graph for some constant d (see Definition 7.31 and Theorem 16.32). 

Let z be a truly random seed of length t = 10 log d(n — k + log1/e) = O(n — k + log1/e). We 
interpret z as a random walk in G of length 10(n — k + log 1/e) starting from the node whose label 
is a. (That is, we think of z as 10(n — k + log1/e) labels in [d] specifying the steps taken in the 
walk.) The output Ext(a, z) of the extractor is the label of the final node on the walk. 

We have ||X — 1113 < ||X ||? = >>, Pr[X = a]?, which is at most 2-* by Lemma 16.41 since X is 
an (n, k)-source. Therefore, after a random walk of length t the distance to the uniform distribution 
is (by the upperbound in (??)): 


1 1 
[MX = Sh < ASX = ¿lla V27 < 2-H? 


When t is a sufficiently large multiple of N — k + log 1/e, this distance is smaller than e. M 


16.6.5 An extractor based upon Nisan-Wigderson 


THIS SECTION IS STILL QUITE ROUGH 

Now we describe an elegant construction of extractors due to Trevisan. 

Suppose we are given a string x obtained from an (N, k)-source. How can we extract k random 
bits from it, given O(log N) truly random bits? Let us check that the trivial idea fails. Using 
2log N random bits we can compute a set of k (where k < N — 1) indices that are uniformly 
distributed and pairwise independent. Maybe we should just output the corresponding bits of x? 
Unfortunately, this does not work: the source is allowed to set N — k bits (deterministically) to 0 so 
long as the remaining k bits are completely random. In that case the expected number of random 
bits in our sample is at most k?/N, which is less than even 1 if k < VN. 

This suggests an important idea: we should first apply some transformation on x to “smear out” 
the randomness, so it is not localized in a few bit positions. For this, we will use error-correcting 
codes. Recall that such codes are used to introduce error-tolerance when transmitting messages 
over noisy channels. Thus intuitively, the code must have the property that it “smears” every bit 
of the message all over the transmitted message. 
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Having applied such an encoding to the weakly random string, the construction selects bits 
from it using a better sampling method than pairwise independent sampling, namely, the Nisan- 
Wigderson combinatorial design. 


Nisan-Wigderson as a sampling method: 


In (??) we defined a function NWy,s(z) using any function f : {0,1} — {0,1} and a com- 
binatorial design S. Note that the definition works for every function, not just hard-to-compute 
functions. Now we observe that NW s(z) is actually a way to sample entries from the truth table 
of f. 

Think of f as a bitstring of length 2', namely, its truth table. (Likewise, we can think of any 
circuit with /-bit inputs and with 0/1 outputs as computing a string of length 2.) Given any z 
(“the seed”), NWy,s(z) is just a method to use z to sample a sequence of m bits from f. This is 
completely analogous to pairwise independent sampling considered above; see Figure ??. 


Figure unavailable in pdf file. 


Figure 16.3: Nisan-Wigderson as a sampling method: An (l, a)-design ($1, .92,...,9m) where each S; C [t], |S: =1 
can be viewed as a way to use z € (0, 1)” to sample m bits from any string of length 2’, which is viewed as the truth 
table of a function f:{0,1}' > {0,1}. 


List-decodable codes 
The construction will use the following kind of codes. 


DEFINITION 16.44 A 
If 6 > 0, a mapping o :(0, iy — {0, pe is called an error-correcting code that is list-decodable up 


to error 1/2 — 6 if for every w € (0, IS, the number of y € BY such that w,o(y) disagree in at 
most 1/2 — ô fraction of bits is at most 1/67. 


The set fola) :x € {0, yy) is called the set of codewords. 


The name “list-decodable” owes to the fact that if we transmit x over a noisy channel after first 
encoding with o then even if the channel flips 1/2 — ô fraction of bits, there is a small “list” of y 
that the received message could be decoded to. (Unique decoding may not be possible, but this will 
be of no consequence in the construction below.) The exercises ask you to prove that list-decodable 
codes exist with N = poly(N, 1/9), where ø is computable in polynomial time. 


Trevisan’s extractor: 

Suppose we are given an (N,k)-source. We fix o : {0,1}% > fot a polynomial-time 
computable code that is list-decodable upto to error 1/2 — e/m. We assume that N is a power 
of 2 and let l = log, N. Now every string x € (Oe may be viewed as a boolean function 
< x >: {0,1}!°&% — {0,1} whose truth table is x. Let S = (S1,...,Sm) be a (l, log m) design over 


[e]. 
The extractor ExtNW : (0, 14" x {0,1} > (0, 1)” is defined as 
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ExtNWg,s(2, 2) = NWo(2)>,8(Z) - 


That is, Ext NW encodes its first (“weakly random”) input x using an error-correcting code, then 
uses Nisan-Wigderson sampling on the resulting string using the second (“truly random”) input z 
as a seed. 


LEMMA 16.45 
For sufficiently large m and for e > gm ExtNW,,s is a (m*, 2e)-extractor. 


PROOF: Let X be an (N, k) source where the min-entropy k is m3. To prove that the distribution 
ExtNW (a, 2) where a € X,z € {0,1} is close to uniform, it suffices (see our remarks after 
Definition 16.38) to show for each function D : {0,1}” — {0,1} that 


Pr,[D(r) = 1] — Prex zeqo,13 [D(ETNW (a, 2)) =1]| < 2e. (18) 


For the rest of this proof, we fix an arbitrary D and prove that (18) holds for it. 

The role played by this test D is somewhat reminiscent of that played by the distinguisher 
algorithm in the definition of a pseudorandom generator, except, of course, D is allowed to be 
arbitrarily inefficient. This is why we will use the black-box version of the Nisan-Wigderson analysis 
(Corollary ??), which does not care about the complexity of the distinguisher. 

Let B be the set of bad a's for this D, where string a € X is bad for D if 


[Pr[D(») = 1] — Pr ¿(0 13 [D(ExtNW(a, z)) =1]| > €. 


We show that B is small using a counting argument: we exhibit a 1-1 mapping from the set of 
bad a's to another set G, and prove G is small. Actually, here is G: 


G = {circuits of size O(m2?)) x {0, ¡Pues 012, 


The number of circuits of size O(m?) is 200 lem), so |G] < 20(m*logm) x 2(m/€)? = 2O(m* logm) 

Let us exhibit a 1-1 mapping from B to G. When a is bad, Corollary ?? implies that there is 
a circuit C of size O(m?) such that either the circuit D(C()) or its negation -XORed with some 
fixed bit b—agrees with o(a) on a fraction 1/2 + €/m of its entries. (The reason we have to allow 
either D(C()) or its complement is the |-| sign in the statement of Corollary ??.) Let w € (0,134 
be the string computed by this circuit. Then o(a) disagrees with w in at most 1/2—e/m fraction of 
bits. By the assumed property of the code a, at most (m/e)? other codewords have this property. 
Hence a is completely specified by the following information: (a) circuit C; this is specified by 
O(m? log m) bits (b) whether to use D(C()) or its complement to compute w, and also the value 
of the unknown bit b; this is specified by 2 bits (c) which of the (m/e)? codewords around w to 
pick as o(a); this is specified by [2log(m/e)] bits assuming the codewords around w are ordered 
in some canonical way. Thus we have described the mapping from B to G. 

We conclude that for any fixed D, there are at most QO(m* logm) bad strings. The probability 
that an element a taken from X is bad for D is (by Lemma ??) at most gn", 90(mélozm) < e for 
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sufficiently large m. We then have 


[Pr [D(r) = 1] — Preex.zepo1y [D(EStNW (a, 2)) = 1] 


IA 


Y Pr[X =q] [Pr[D(r) = 1] — Pr efo 13 [D(ExtNW (a, 2) = 1] 
< Pr[X e B]+e<2e, 


where the last line used the fact that if a Z B, then by definition of B, 
Pr[D(r) = 1] — Pr croy [D(EXtNWw (a, z))=1]|<e. m 


The following theorem is an immediate consequence of the above lemma. 


THEOREM 16.46 
Fix a constant e; for every N and k= N°) there is a polynomial-time computable (k, e) -extractor 


Ext : (0,14% x (0,1) — {0,1} where m = k" and t = O(log N). 


16.7 Applications of Extractors 


Extractors are deterministic objects with strong pseudorandom properties. We describe a few 
important uses for them; many more will undoubtedly be found in future. 


16.7.1 Graph constructions 


An extractor is essentially a graph-theoretic object; see Figure ??. (In fact, extractors have been 
used to construct expander graphs.) Think of a (k, e) extractor Ext : {0,1} x {0,1}! — {0,1} 
as a bipartite graph whose left side contains one node for each string in {0, ie and the right side 
contains a node for each string in {0,1}. Each node a on the left is incident to 2° edges, labelled 
with strings in {0,1}, with the right endpoint of the edge labeled with z being Ext(a, z). 

An (N, k)-source corresponds to any distribution on the left side with min-entropy at least k. 
The extractor’s definition implies that picking a node according to this distribution and a random 
outgoing edge gives a node on the right that is essentially uniformly distributed. 


Figure unavailable in pdf file. 


Figure 16.4: An extractor Ext : [0,14% x {0,1}7 — {0,1} defines a bipartite graph where every node on the left 
has degree 27. 


This implies in particular that for every set X on the left side of size exactly 2" —notice, this is 
a special case of an (N, k)-source— its neighbor set T (X) on the right satisfies |[(X)| > (1 — €)2”. 

One can in fact show a converse, that high expansion implies that the graph is an extractor; 
see Chapter notes. 
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16.7.2 Running randomized algorithms using weak random sources 


We now describe how to use extractors to simulate probabilistic algorithms using weak random 
sources. Suppose that A(-,-) is a probabilistic algorithm that on an input of length n uses m = m(n) 
random bits, and suppose that for every x we have Pr,|A(z,r) = right answer ] > 3/4. If A’s 
answers are 0/1, then such algorithms can be viewed as defining a BPP language, but here we 
allow a more general scenario. Suppose Ext : {0,1} x (0,1) > {0,1}” is a (k, 1/4)-extractor. 

Consider the following algorithm A’: on input x € {0,1}" and given a string a € {0,1}% 
from the weakly random source, the algorithm enumerates all choices for the seed z and computes 
A(x, Ext(a, z). Let 


A'(x, a) = majority value of { A(z, Ext(a, z)) : z € {0, 1}"} (19) 


The running time of A’ is approximately 2% times that of A. We show that if a comes from an 
(n,k + 2) source, then A’ outputs the correct answer with probability at least 3/4. 

Fix the input x. Let R = {r € {0,1}: A(a,r) = right answer }, and thus |R| > 32”. Let 
B be the set of strings a € {0, qa for which the majority answer computed by algorithm A’ is 
incorrect, namely, 


B= fa : Pr, ¿19 1 [Az, Ext(a, z)) = right answer] < 1/2} 


= fa : Pr ero, 1} [Ext(a, 2) € R] < 1/2} 


CLAIM: |B| < 2%, 
Let random variable Y correspond to picking an element uniformly at random from B. Thus Y 
has min-entropy log B, and may be viewed as a (N, log B)-source. By definition of B, 


Prey zeto 1 [ExXt(a, 2) € R] < 1/2. 
But |R| = 32m, so we have 
Pr icy zeto, y [EXt(a, 2) = R] z Pr,e{0,1}” [r € R] > 1/4, 


which implies that the statistical distance between the uniform distribution and Ext(Y, z) is at least 
1/4. Since Ext is a (k,1/4)-extractor, Y must have min-entropy less than k. Hence |B| < 2 and 
the Claim is proved. 

The correctness of the simulation now follows since 


Prauex[4'(x, a) = right answer ] = 1 — Praexla € B] 
> 1-27 (+2) .|B] > 3/4, (by Lemma ?7). 


Thus we have shown the following. 
THEOREM 16.47 
Suppose A is a probabilistic algorithm running in time Ta(n) and using m(n) random bits on 
inputs of length n. Suppose we have for every m(n) a construction of a (k(n), 1/4)-extractor 
Extn : {0,1}% x (0,149 — {0,1} running in Tg(n) time. Then A can be simulated in time 
2'(T4 + Tg) using one sample from a (N,k +2) source. 
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16.7.3 Recycling random bits 


We addressed the issue of recycling random bits in Section ??. An extractor can also be used to 
recycle random bits. (Thus it should not be surprising that random walks on expanders, which 
were used to recycle random bits in Section ??, were also used to construct extractors above.) 

Suppose A be a randomized algorithm that uses m random bits. Let Ext : {0,1} x (0, i? = 
{0,1} be any (k,e)-extractor. Consider the following algorithm. Randomly pick a string a € 
(0,13%, and obtain 2 strings in (0,1)” obtained by computing Ext(a, z) for all z € {0,1}*. Run 
A for all these random strings. Note that this manages to run A as many as 2t times while using 
only N random bits. (For known extractor constructions, N < 2'm, so this is a big saving.) 

Now we analyse how well the error goes down. Suppose D C {0,1} be the subset of strings 
for which A gives the correct answer. Let p = |D| /2””; for a BPP algorithm p > 2/3. Call an 
a € {0,1} bad if the above algorithm sees the correct answer for less than p — e fraction of 2’s. If 
the set of all bad a’s were to have size more than 2", the (N, k)-source X corresponding to drawing 
uniformly at random from the bad a’s would satisfy 


Pr[Ext(X, U+) € D] — Pr[Um € D] > e, 


which would contradict the assumption that Ext is a (k, e)-extractor. We conclude that the prob- 
ability that the above algorithm gets an incorrect answer from A in p — e fraction of the repeated 
runs is at most 2*/2%. 


16.7.4 Pseudorandom generators for spacebounded computation 


Now we describe Nisan’s pseudo-random generators for space-bounded randomized computation, 
which allows randomized logspace computations to be run with O(log? n) random bits. 

Throughout this section we represent logspace machines by their configuration graph, which has 
size poly(n). 


THEOREM 16.48 (NISAN) 
. . . . clog? n nd 
For every d there is ac > 0 and a polynomial-time computable function g:{0,1}°°® " — {0,1} 
such that for every space-bounded machine M that has a configuration graph of size < nt on inputs 
of size n: 
Pr [M(x,r)=1]- Pr [M(zx, g(2)) =1]| < En (20) 
refo,1p z€{0,1}¢1087 n 10 
We give a proof due to Impagliazzo, Nisan, and Wigderson [INW94] (with further improvements 
by Raz and Reingold [RR99]) that uses extractors. Nisan's original paper did not explicitly use 
extractors —the definition of extractors came later and was influenced by results such as Nisan’s. 
In fact, Nisan's construction proves a result stronger than Theorem 16.48: there is a polynomial- 
time simulation of every algorithm in BPL using O(log?) space. (See Exercises.) Note that 
Savitch’s theorem (Theorem ??) also implies that BPL C SPACE(log?n), but the algorithm 
in Savitch’s proof takes n!%8” time. Saks and Zhou [SZ99a] improved Nisan's ideas to show that 
BPL C SPACE(log!* n), which leads many experts to conjecture that BPL = L (i.e., randomness 
does not help logspace computations at all). (For partial progress, see Section ?? later.) 
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The main intuition behind Nisan's construction —and also the conjecture BPL = L— is that 
the logspace machine has one-way access to the random string and only O(log n) bits of memory. 
So it can only “remember” O(logn) of the random bits it has seen. To exploit this we will use 
the following simple lemma, which shows how to recycle a random string about which only a little 
information is known. (Throughout this section, o denotes concatenation of strings.) 


LEMMA 16.49 (RECYCLING LEMMA) 
Let f :{0,1}" — {0,1}* be any function and Ext: {0,1}" x {0,1}' — {0,1}” be a (k, €/2)-extractor, 
where k =n —(s+1)—log+. When X Ep ([0,1)”, W Er {0,1}, z Er (0, 1}, then 


f(X)oW & f(X) o Ext(X, 2). 


REMARK 16.50 

When the lemma is used, s < n and n = m. Thus f(X), which has length s, contains only a small 
amount of information about X. The Lemma says that using an appropriate extractor (whose 
random seed can have length as small as t = O(s + log(1/e)) if we use Lemma 16.43) we can get a 
new string Ext(X, z) that looks essentially random, even to somebody who knows f(X). 


PROOF: For v € {0,1}° we denote by X, the random variable that is uniformly distributed over 
the set f7*(w). Then we can express || (f(X) o W — f(X) o Ext(X, z) || as 


= 5 Y Prt) = v AW = w) — Pr[f(X) = v A EXX, 2) = w) 
= Y Prlf(X) = v]: |W - Ext(X,, 2) | (21) 


v 


Let V = {v : Pr[f(X) = v] > €/2°t!}. Ifv € V, then we can view X, as a (n,k)-source, where 
k > n-— (s + 1) — log L, Thus by definition of an extractor, Ext(X,,r) =e;2 W and hence the 
contributions from v € V sum to at most e/2. The contributions from v ¢ V are upperbounded by 
>ugv Pri f(X) = v] < 2° x zr = €/2. The lemma follows. M 


Now we describe how the Recycling Lemma is useful in Nisan’s construction. Let M be a 
logspace machine. Fix an input of size n and view the graph of all configurations of M on this 
input as a leveled branching program. For some d > 1, M has < n? configurations and runs in time 
L < nt. Assume without loss of generality —since unneeded random bits can always be ignored— 
that it uses 1 random bit at each step. Without loss of generality (by giving M a separate worktape 
that maintains a time counter), we can assume that the configuration graph is leveled: it has L 
levels, with level ¿ containing configurations obtainable at time i. The first level contains only 
the start node and the last level contains two nodes, “accept” and “reject;” every other level has 
W = n? nodes. Each level i node has two outgoing edges to level i + 1 nodes and the machine’s 
computation at this node involves using the next bit in the random string to pick one of these two 
outgoing edges. We sometimes call L the length of the configuration graph and W the width. 

For simplicity we first describe how to reduce the number of random bits by a factor 2. Think 
of the L steps of the computation as divided in two halves, each consuming L/2 random bits. 
Suppose we use some random string X of length L/2 to run the first half, and the machine is now 
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Figure 16.5: Configuration graph for machine M 


at node v in the middle level. The only information known about X at this point is the index of 
v, which is a string of length dlogn. We may thus view the first half of the branching program 
as a (deterministic) function that maps {0,1}”/? bits to (0,1187 bits. The Recycling Lemma 
allows us to use a random seed of length O(logn) to recycle X to get an almost-random string 
Ext(X, z) of length L/2, which can be used in the second half of the computation. Thus we can run 
L steps of computation using L/2 + O(logn) bits, a saving of almost a factor 2. Using a similar 
idea recursively, Nisan’s generator runs L steps using O(log n log L) random bits. 
Now we formally define Nisan’s generator. 


DEFINITION 16.51 (NISAN’S GENERATOR) 

For some r > 0 let Ext,: {0,1}*" x {0,1}” > {0,1}*" be an extractor function for each k > 0. For 
every integer k > 0 the associated Nisan generator Gy : {0,1}*" = {0, 1}2" is defined recursively 
as (where |a| = (k — 1)r,|z| = r) 


zı (i.e., first bit of z) k=1 
Gglao z) = 
Gr_1(a) o Gx-1(Extx-1(a,2)) k>1 


Now we use this generator to prove Theorem 16.48. We only need to show that the probability 
that the machine goes from the start node to the “accept” node is similar for truly random strings 
and pseudorandom strings. However, we will prove a stronger statement involving intermediate 
steps as well. 

If nodes u is a node in the configuration graph, and s is a string of length 2%, then we denote by 
fu,2x (5) the node that the machine reaches when started in u and its random string is s. Thus if s 
comes from some distribution D, we can define a distribution f,, (D) on nodes that are 2* levels 
further from u. 

THEOREM 16.52 

Let r = O(logn) be such that for each k < dlogn, Ext, : {0,1}*" x {0,1}" > {0,1}*" is a 
(kr — 2dlog n, e)-extractor. For every machine of the type described in the previous paragraphs, 
and every node u in its configuration graph: 


Il fu,2 (Uze) — fa, 2 (GeUrr)) I< 3%e, (22) 


where U; denotes the uniform distribution on (0,1Y'. 


REMARK 16.53 

To prove Theorem 16.48 let u = uo, the start configuration, and 2" = L, the length of the entire 
computation. Choose 3Fe < 1/10 (say), which means log 1/e = O(log L) = O(logn). Using the 
extractor of Section 16.6.4 as Ext, we can let r = O(log n) and so the seed length kr = O(r log L) = 
O(log? n). 
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PROOF: (Theorem 16.52) Let ez denote the maximum value of the left hand side of (22) over all 
machines. The lemma is proved if we can show inductively that e, < 2e,_; + 2e. The case k = 1 
is trivial. At the inductive step, we need to upperbound the distance between two distributions 
For (D1), f. 21 (Da), for which we introduce two distributions D2, D3 and use triangle inequality: 


3 
ll fuar (D1) — fuze (Da) ISO Il Lua (Di) — fu 2 (Pina) Il - (23) 


¿=1 
The distributions will be: 


D= Ua 

Da = Ge Ver) 

Dz = Ugr-1 o Gr-1(U(r—1)r) 

D3 = Ge_1(U(e-1)r) © Gra (U(g-1)) (U, U” are identical but independent). 


We bound the summands in (23) one by one. 
Claim 1: || fuo*(D1) — fu ar (Da) IS €n-1- 
Denote Pr[fu 2-1 (Uzx-1) = w] by puw and Prlf,, 1-1 (Gg-1(U(-1)r)) = w] by quw. According to 
the inductive assumption, 


1 
5 Y [Puw — Quaol =|| La, or (Woe) — fat (Gr-1(U-1)r)) IIS ea 


Since Dı = Ux may be viewed as two independent copies of Uzx-1 we have 


1 
I fuar (D1) = ugk (D2) | = > 2 Y Patan = Puedo 
v w w 


2k-! and 2* levels respectively from u 


= ) e > |[Pwv = qwl 
2 
w v 


< ex-1 (using inductive hypothesis and > Puw = 1) 


w 


where w, v denote nodes 


Claim 2: || fur (D2) — for (D3) IS €n-1- 

The proof is similar to the previous case. 

Claim 3: | fuar (D3) = fuar (Da) |< Ze. 

We use the Recycling Lemma. Let gu : (0, 10" — [1, W] be defined as g,(a) = Fu,gr-1(Ge_1(a)). 
(To put it in words, apply the Nisan generator to the seed a and use the result as a random string 


for the machine, using u as the start node. Output the node you reach after 2*7! steps.) Let 
X,Y € U1), and z € Up. According to the Recycling Lemma, 


gu(X) o Y Re gu(X) o Extn i(X, 2), 
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and then part 3 of Lemma 16.39 implies that the equivalence continues to hold if we apply a 
(deterministic) function to the second string on both sides. Thus 


Jul X) Q Iwl Y) Te Iul X) o Jw(Extk-1(X, z)) 


for all nodes w that are 2%7t levels after u. The left distribution corresponds to f., 2» (D3) (by which 
we mean that Pr[f,,9x(D3) = v] =>, PrlgulX) = w A^ gw(Y) = v]) and the right one to fy or (Da) 
and the proof is completed. Mi 


Chapter notes and history 


The results of this section have not been presented in chronological order and some important 
intermediate results have been omitted. Yao [Yao82] first pointed out that cryptographic pseudo- 
random generators can be used to derandomize BPP. A short paper of Sipser [Sip88] initiated 
the study of “hardness versus randomness,” and pointed out the usefulness of a certain family of 
highly expanding graphs that are now called dispersers (they are reminiscent of extractors). This 
research area received its name as well as a thorough and brilliant development in a paper of Nisan 
and Wigderson [NW94]. MISSING DISCUSSION OF FOLLOWUP WORKS TO NW94 

Weak random sources were first considered in the 1950s by von Neumann [von61]. The second 
volume of Knuth’s seminal work studies real-life pseudorandom generators and their limitations. 
The study of weak random sources as defined here started with Blum [Blu84]. Progressively weaker 
models were then defined, culminating in the “correct” definition of an (N, k) source in Zucker- 
man [Zuc90]. Zuckerman also observed that this definition generalizes all models that had been 
studied to date. (See [SZ99b] for an account of various models considered by previous researchers. ) 
He also gave the first simulation of probabilistic algorithms with such sources assuming k = Q(N). 
A succession of papers has improved this result; for some references, see the paper of Lu, Rein- 
gold, Vadhan, and Wigderson [LRVW03], the current champion in this area (though very likely 
dethroned by the time this book appears). 

The earliest work on extractors —in the guise of leftover hash lemma of Impagliazzo, Levin, 
and Luby [ILL89] mentioned in Section 16.6.3— took place in context of cryptography, specifically, 
cryptographically secure pseudorandom generators. Nisan [Nis92] then showed that hashing could 
be used to define provably good pseudorandom generators for logspace. 

The notion of an extractor was first formalized by Nisan and Zuckerman [NZ96]. Trevisan [Tre01] 
pointed out that any “black-box” construction of a pseudorandom generator gives an extractor, and 
in particular used the Nisan-Wigderson generator to construct extractors as described in the chap- 
ter. His methodology has been sharpened in many other papers (e.g.,see [LRVW03}). 

Our discussion of derandomization has omitted many important papers that successively im- 
proved Nisan-Wigderson and culminated in the result of Impagliazzo and Wigderson [[W01]that 
either NEXP = BPP (randomness is truly powerful!) or BPP has an a subexponential “simula- 
tion.” 7 Such results raised hopes that we were getting close to at least a partial derandomization 
of BPP, but these hopes were dashed by the Impagliazzo-Kabanets [K103] result of Section 16.3. 


"The “simulation” is in quotes because it could fail on some instances, but finding such instances itself requires 
exponential computational power, which nature presumably does not have. 
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Trevisan’s insight about using pseudorandom generators to construct extractors has been greatly 
extended. It is now understood that three combinatorial objects studied in three different fields 
are very similar: pseudorandom generators (cryptography and derandomization), extractors (weak 
random sources) and list-decodable error-correcting codes (coding theory and information theory). 
Constructions of any one of these objects often gives constructions of the other two. For a survey, 
see Vadhan’s lecture notes [?]. 

STILL A LOT MISSING 

Expanders were well-studied for a variety of reasons in the 1970s but their application to 
pseudorandomness was first described by Ajtai, Komlos, and Szemeredi [AKS87]. Then Cohen- 
Wigderson [CW89] and Impagliazzo-Zuckerman (1989) showed how to use them to “recycle” ran- 
dom bits as described in Section 7.B.3. The upcoming book by Hoory, Linial and Wigderson (draft 
available from their web pages) provides an excellent introduction to expander graphs and their 
applications. 

The explicit construction of expanders is due to Reingold, Vadhan and Wigderson [RVW00], 
although we chose to present it using the replacement product as opposed to the closely related 
zig-zag product used there. The deterministic logspace algorithm for undirected connectivity is due 
to Reingold [?]. 


Exercises 


$1 Verify Corollary 16.6. 


$2 Show that there exists a number e > 0 and a function G : {0,1}* — {0,1}* that satisfies all of 
the conditions of a 2‘-pseudorandom generator per Definition ??, save for the computational 
efficiency condition. 
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§3 Show by a counting argument (i.e., probabilistic method) that for every large enough n there 
is a function f:(0,1)” — {0,1}, such that Huf) > 29/10, 


§4 Prove that if there exists f € E and e > 0 such that H,¿(f)(n) > 27 for every n € N, then 
MA = NP. 


85 We define an oracle Boolean circuit to be a Boolean circuit that have special gates with 
unbounded fanin that are marked ORACLE. For a Boolean circuit C and language O C {0,1}", 
we define by CO (x) the output of C on x, where the operation of the oracle gates when fed 
input q is to output 1 iff q € O. 


(a) Prove that if every f € E can be computed by a polynomial-size circuits with oracle to 
SAT, then the polynomial hierarchy collapses. 

(b) For a function f : {0,1}* — {0,1} and O C {0,1}*, define H,O(f) to be the function 
that maps every n € N to the largest S such that Prep (0,1)" [09 (2) = f(x)] < 1/2+1/S. 


16.7. 
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$11 


$12 


813 


$14 


815 


Prove Lemma 16.39. 


Prove that for every function Ext : {0,1}” — {0,1} and there exists an (n,n — 1)-source X 
and a bit b € {0,1} such that Pr[Ext(X)¡ = b] = 1 (where Ext(X), denotes the first bit of 
Ext(X)). Prove that this implies that ó(Ext(X), Um) > 1/2. 


Show that there is a constant c > 0 such that if an algorithm runs in time T and requires 
m random bits, and m > k + clog T, then it is not possible in general to simulate it in a 
blackbox fashion using an (N, k) source and O(log n) truly random bits. 
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A flat (N, k) source is a (N, k) source where for every x € (0,1) pz is either 0 or exactly 
e 

Show that a source X is an (N, k)-source iff it is a distribution on flat sources. In other words, 
there is a set of flat (N, k)-sources X1, X2,... and a distribution D on them such that drawing 
a sample of X corresponds to picking one of the X;’s according to D, and then drawing a 
sample from X;. 
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Use Nisan's generator to give an algorithm that produces universal traversal sequences for 
n-node graphs (see Definition ??) in n*%87)_-time and O(log? n) space. 


Suppose boolean function f is (S, e)-hard and let D be the distribution on m-bit strings defined 
by picking inputs 271,22,...,2%m uniformly at random and outputting f(71)f(x2)--- f(m). 
Show that the statistical distance between D and the uniform distribution is at most em. 


Prove Lemma 16.42. 


(Klivans and van Melkebeek 1999) Suppose the conclusion of Lemma ?? is true. Then show 
that MA C i.o.—[NTIME(2”) /n]. 

(Slightly harder) Show that if NEXP #4 EXP then AM C ¡.0.—[NTIME(2”)/n]. 

Let A be an n x n matrix with eigenvectors u!,...,u” and corresponding values Ay,..., An. 


Let B be an mx m matrix with eigenvectors v!,...,v'” and corresponding values a1,...,Qm- 
Prove that the matrix A & B has eigenvectors u’ & v’ and corresponding values A; - a;. 


Prove that for every two graphs G, G', MG8G”) < A(G) + A(G") without using the fact that 
every symmetric matrix is diagonalizable. 
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$16 Let G be an n-vertex D-degree graph with p combinatorial edge expansion for some p > 0. 
(That is, for every a subset S of G’s vertices of size at most n/2, the number of edges 
between S and its complement is at least pd|S|.) Let G” be a D-vertex d-degree graph with 
p' combinatorial edge expansion for some p’ > 0. Prove that G@®G’ has at least p?p'/1000 
edge expansion. 


* 0 Jo uorsuedxo oy} 9sn 19338] 973 
10} ƏM ‘H Jo uotsuedxo oy} 9SN IQULIOJ oY} 104 JEY} ueyy sso] dn 
aye} yey} 98047 pue sio3sn]o 1oy} jo uorod QT /d — 7 uey} orou 
dn oye yey} syosqns oy} ÁJJUƏIƏ IP ye91], 'SIOJSNJO [enprarput ƏY 
jo sjosqns u se jo JU3noyy 9q ued 0) (8 H jo yosqns Ag JWH 


Acknowledgements 


We thank Luca Trevisan for cowriting an early draft of this chapter. Thanks also to Valentine 
Kabanets, Omer Reingold, and lannis Tourlakis for their help and comments. 


Chapter 17 


Hardness Amplification and Error 
Correcting Codes 


We pointed out in earlier chapters (e.g., Chapter ?? the distinction between worst-case hardness 
and average-case hardness. For example, the problem of finding the smallest factor of every given 
integer seems difficult on worst-case instances, and yet is trivial for at least half the integers — 
namely, the even ones. We also saw that functions that are average-case hard have many uses, 
notably in cryptography and derandomization. 

In this chapter we study techniques for amplifying hardness. First, we see Yao’s XOR Lemma, 
which transforms a “mildly hard” function (i.e., one that is hard to compute on a small fraction 
of the instances) to a function that is extremely hard, for which the best algorithm is as bad as 
the algorithm that just randomly guesses the answer. We mentioned Yao’s result in the chapter 
on cryptography as a means to transform weak one-way functions into strong one-way functions. 
The second result in this chapter is a technique to use error-correcting codes to transform worst- 
case hard functions into average-case hard functions. This transformation unfortunately makes the 
running time exponential, and is thus useful only in derandomization, and not in cryptography. 

In addition to their applications in complexity theory, the ideas covered here have had other 
uses, including new constructions of error-correcting codes and new algorithms in machine learning. 


17.1 Hardness and Hardness Amplification. 


We now define a slightly more refined notion of hardness, that generalizes both the notions of 
worst-case and average-case hardness given in Definition 16.7: 


DEFINITION 17.1 (HARDNESS) 

Let f : {0,1}" — {0,1} and p : N — [0,1]. We define H£,(f) to be the func- 
tion from N to N that maps every number n to the largest number S such that 
Presento 1yr[Clz) = f(x)] < p(n) for every Boolean circuit C on n inputs with size 


at most S. 


p17.1 (321) 
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Note that, in the notation of Definition 16.7, Hed f) = Hi,(f) and H,,{f)(n) = max {S : H1/2+1/5 £) (n) > S}. 


avg 


In this chapter we show the following results for every two functions S, S’ : N — N: 


Worst-case to mild hardness. If there is a function f € E = DTIME(29()) such that H,,.(f)(n) = 
HI) (nm) > S(n) then there is a function f’ € E such that H2.°%f)(n) > S(en)" for some 


avg 


constant e > 0 and every sufficiently large n. 


Mild to strong hardness. If f' € E satisfies H?.°%( f’)(n) > S'(n) then there is f” € E and e > 0 
such that H,{f”)(n) > S nos 


Combining these two results with Theorem 16.10, this implies that if there exists a function 
FEE with H,Lf) (nm) > S(n) then there exists an S(¢°)‘-pseudorandom generator for some e > 0, 
and hence: 


Corollary 1 If there exists f € E and e > 0 such that H,.Lf) > 2% then BPP C QuasiP = 
U¿DTIME(2%8""), 


Corollary 2 If there exists f € E such that H,,(f) > n“) then BPP C SUBEXP =N.DTIME(2”). 


To get to BPP = P, we need a stronger transformation. We do this by showing how to 
transform in one fell swoop, a function f € E with H,,(f) > S(n) into a function f’ € E with 
Hud f) > S(en)© for some e > 0. Combined with Theorem 16.10, this implies that BPP = P if 
there exists f € E with H,,(f) > 2°). 


17.2 Mild to strong hardness: Yao’s XOR Lemma. 


We start with the second result described above: transforming a function that has “mild” average- 
case hardness to a function that has strong average-case hardness. The transformation is actually 
quite simple and natural, but its analysis is somewhat involved (yet, in our opinion, beautiful). 


THEOREM 17.2 (YAO’s XOR LEMMA) 

For every f : {0,1}" > {0,1} and k € N, define f®* : (0,1)”* > {0,1} as follows: 
FO" isan ig) = Ns Fi) (mod 2): 

For every ô > 0, S and e > 2(1 — 6/2)", if Hf) > S then 


avg 


H1/2+< FOK) > 


2 
€ 
avg = 100 log(1/de) S 


The intuition behind Theorem 17.2 derives from the following fact. Suppose we have a biased 
coin that, whenever it is tossed, comes up heads with probability 1 — ô and tails with probability 
6. If 6 is small, each coin toss is fairly predictable. But suppose we now toss it k times and define 
a composite coin toss that is “heads” iff the coin came up heads an odd number of times. Then 
the probability of “heads” in this composite coin toss is at most 1/2 + (1 — 26)* (see Exercise 1), 
which tends to 1/2 as k increases. Thus the parity of coin tosses becomes quite unpredictable. The 
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analogy to our case is that intuitively, for each i, a circuit of size S has chance at most 1 — 6 of 
“knowing” f(a;) if x; is random. Thus from its perspective, whether or not it will be able to know 
f(a;) is like a biased coin toss. Hence its chance of guessing the parity of the k bits should be 
roughly like 1/2 + (1 — 29)". 

We transform this intuition into a proof via an elegant result of Impagliazzo, that provides some 
fascinating insight on mildly hard functions. 


DEFINITION 17.3 (0-DENSITY DISTRIBUTION) 
For 6 < 1 a 6-density distribution H over {0,1}" is one such that for every x € ([0,1)”, Pr[H = 


a] < 27. 


REMARK 17.4 
Note that in Chapter 16 we would have called it a distribution with min entropy n — log 1/6. 

The motivating example for this definition is the distribution that is uniform over some subset 
of size 62” and has O probability outside this set. 


A priori, one can think that a function f that is hard to compute by small circuits with 
probability 1 — ô could have two possible forms: (a) the hardness is sort of “spread” all over the 
inputs, and it is roughly 1 — ó-hard on every significant set of inputs or (b) there is a subset H 
of roughly a 6 fraction of the inputs such that on H the function is extremely hard (cannot be 
computed better than 5 +e for some tiny e) and on the rest of the inputs the function may be even 
very easy. Such a set may be thought of as lying at the core of the hardness of f and is sometimes 
called the hardcore set. Impagliazzo’s Lemma shows that actually every hard function has the form 
(b). (While the Lemma talks about distributions and not sets, one can easily transform it into a 
result on sets.) 


LEMMA 17.5 (IMPAGLIAZZO’S HARDCORE LEMMA) 
For every ô > 0, f : {0,1}" — {0,1}”, and e > 0, if Hi, f) > S then there exists a distribution H 


over [0,1)” of density at least 6/2 such that for every circuit C of size at most aa" 


oe = f(x)] < /2+e, 


Proof of Yao’s XOR Lemma using Impagliazzo’s Hardcore Lemma. 


We now use Lemma 17.5 to transform the biased-coins intuition discussed above into a proof of the 
XOR Lemma. Let f : (0,1)” — {0,1} be a function such that HIX f) > S, let k € N and suppose, 


for the sake of contradiction, that there is a circuit C of size nó such that 


k 
Pr ACTA => fe (mod 2) > BES, (1) 
(21,...,14)ERUE i=1 
where e > 2(1 — 9/2)". 
Let H be the hardcore distribution of dens ity at least 6’ = 9/2 that is obtained from Lemma 17.5, 


on which every circuit C’ fails to compute f with probability better than 1/2 + €/2. Define a dis- 
tribution G over {0,1}" as follows: for every x € {0,1}", Pr[G = a] = (1 — 0’ Pr[H = 2))/(1 — ô’). 
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Note that G is indeed a well-defined distribution, as H has density at least 6’. Also note that if H 
was the uniform distribution over some subset of {0,1}” of size 0/2”, then G will be the uniform 
distribution over the complement of this subset. 

We can think of the process of picking a uniform element in {0,1}” as follows: first toss a 
6’-biased coin that comes up “heads” with probability ô. Then, if it came up “heads” choose a 
random element out of H, and with probability 1 — 6’, and otherwise choose a random element out 
of G. We shorthand this and write 


Un =(1-9)G+8'H. (2) 


If we consider the distribution (Un)? of picking two random strings, then by (2) it can be written 
as (1 — 6')*G? + (1 — 98'GH + 9'(1— 0) HG + 92H?. Similarly, for every k 


(Un)* =(1-8)G6*+ (1-0 1G H+... + 58H. (3) 
For every distribution D over {0, ea let Pp be the probability of the event of the left-hand side of 


(1) that Clas. £k) = oe f(a) (mod 2) where 1,..., £p are chosen from D. Then, combining 
(1) and (3), 


Ya + € < Py ye = (1 — 8 Par + (1-0) Por-ig + +O Pipe . 
But since 6’ = 6/2 and e > 2(1 — 6/2)* and Por < 1 we get 
1/2 + €/2 < 2+ € — (1 — 9)" < (1 — WW Paap tot 8 Pyr . 
Notice, the coefficients of all distributions on the right hand side sum up to less than one, so there 


must exist a distribution D that has at least one H component such that Pp > 1/2 + €/2. Suppose 
that D = G*-1H (all other cases are handled in a similar way). Then, we get that 


k 


[C(X1,..., Xk- Xp) = Y F(X) (mod 2)] > 1/2 + €/2. (4) 


Pr 
Xi. Xk-1ERG,XkERH i=1 


By the averaging principle, (4) implies that there exist k — 1 strings 21,...,1,-1 such that if 
b= yea (xi) (mod 2) then, 


Pr [Clzxi,...,Tg-1, Xx) =b+ f(X~) (mod 2)] > 1/2 + e/2. (5) 
XkERH 
But by “hardwiring” the values z1,...,£ẹ and b into the circuit C, (5) shows a direct contradiction 


to the fact that H is a hardcore distribution for the function f. E 


17.3 Proof of Impagliazzo’s Lemma 


Let f be a function with HIX f) > S. To Prove Lemma 17.5 we need to show a distribution H over 
{0,1}” (with no element of weight more than 2-27~"/6) on which every circuit C of size S’ cannot 
compute f with probability better than 1/2 + e (where S’,¢ are as in the Lemma’s statement). 
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Let’s think of this task as a game between two players named Russell and Noam. Russell first 
sends to Noam some distribution H over {0,1}" with density at least 6. Then Noam sends to 
Russell some circuit C of size at most S’. Russell then pays to Noam Ese [Righto(x)] dollars, 
where Righto(x) is equal to 1 if C(x) = f(x) and equal to 0 otherwise. What we need to prove is 
that there is distribution that Russell can choose, such that no matter what circuit Noam sends, 
Russell will not have to pay him more than 1/2 + e dollars. 

An initial observation is that Russell could have easily ensured this if he was allowed to play 
second instead of first. Indeed, under our assumptions, for every circuit C of size S (and so, in 
particular also for circuits of size S” which is smaller than S), there exists a set Sc of at least 
92” > (9/2)2” inputs such that C(x) 4 f(x) for every x € Sc. Thus, if Noam had to send his 
circuit C, then Russell could have chosen H to be the uniform distribution over Sc. Thus H would 
have density at least 9/2 and E,ze,4[Right¢(x)] = 0, meaning that Russell wouldn't have to pay 
Noam a single cent. 

Now this game is a zero sum game, since whatever Noam gains Russell loses and vice versa, 
tempting us to invoke von-Neumann's famous Min-Maz Theorem (see Note 17.7) that says that 
in a zero-sum game it does not matter who plays first as long as we allow randomized strategies.' 
What does it mean to allow randomized strategies in our context? It means that Noam can 
send a distribution C over circuits instead of a single circuit, and the amount Russell will pay is 
EcepcErepH|Rightc(x)]. (It also means that Russell is allowed to send a distribution over 6/2- 
density distributions, but this is equivalent to sending a single 6/2-density distribution.) 

Thus, we only need to show that, when playing second, Russell can still ensure a payment 
of at most 1/2 + e dollars even when Noam sends a distribution C of S'-sized circuits. For every 
distribution C, we say that an input x € {0,1}” is good for Noam (good for short) with respect to 
C if Ecenc[Righto(x)] > 1/2+ €. It suffices to show that for every distribution C over circuits of size 
at most S’, the number of good 2's with respect to C is at most 1 — 6/2. (Indeed, this means that 
for every C, Russell could choose as its distribution H the uniform distribution over the bad inputs 
with respect to C.) 

Suppose otherwise, that there is at least a 1 — 6/2 fraction of inputs that are good for C. We 
will use this to come up with an S-sized circuit C that computes f on at least a 1 — 6 fraction 
of the inputs in {0,1}”, contradicting the assumption that HIZA f) > S. Let t = 10log(1/de) /e?, 
choose C1,...,C at random from C and let C = maj{C),...,C;} be the circuit of size tS" < S 
circuit that on input x outputs the majority value of {C\(x),...,C:(x)}. If x is good for C, then 
by the Chernoff bound we have that C(x) = f(x) with probability at least 1 — 0/2 over the choice 
of C1,..., Ci. Since we assume at least 1 — 6/2 of the inputs are good for C, we get that 


Exento," ECE RC,...Crenc [Rightmai(cy.,...,.c:}(2)] = (1 Da 2) >1-—ô. (6) 


But by linearity of expectation, we can switch the order of expectations in (6) obtaining that 


ECiERC,... CrerCExep{0,1}” [Rightmajto,. cg) 21-4, 


'The careful reader might note that another requirement is that the set of possible moves by each player is finite, 
which does not seem to hold in our case as Russell can send any one of the infinitely many 6/2-density distributions. 
However, by either requiring that the probabilities of the distribution are multiples of 395-57 (which won’t make any 
significant difference in the game’s outcome), or using the fact that each such distribution is a convex sum of uniform 


distributions over sets of size at least (9/2)2” (see Exercise 9 of Chapter 16), we can make this game finite. 
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which in particular implies that there exists a circuit C of size at most S such that Eze yu, [Righto(x)] > 
1 — 6, or in other words, C computes f on at least a 1 — ô fraction of the inputs. W 


REMARK 17.6 

Taken in the contrapositive, Lemma 17.5 implies that if for every significant chunk of the inputs 
there is some circuit that computes f with on this chunk with some advantage over 1/2, then there 
is a single circuit that computes f with good probability over all inputs. In machine learning such 
a result (transforming a way to weakly predict some function into a way to strongly predict it) is 
called Boosting of learning methods. Although the proof we presented here is non-constructive, 
Impagliazzo’s original proof was constructive, and was used to obtain a boosting algorithm yielding 
some new results in machine learning, see [?]. 
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NOTE 17.7 (THE MIN-MAX THEOREM) 

A zero sum game is, as the name implies, a game between two parties in 
which whatever one party loses is won by the other party. It is modeled 
by an m x n matrix A = (a; j) of real numbers. The game consists of only 
a single move. One party, called the minimizer or column player, chooses 
an index j € [n] while the other party, called the mazimizer or row player, 
chooses an index i € [m]. The outcome is that the column player has to pay 
ai j units of money to the row player (if a; ¿ is negative then actually the row 
player has to pay). Clearly, the order in which players make their moves is 
important. Surprisingly, if we allow the players randomized strategies, then 
the order of play becomes unimportant. 


The game with randomized (also known as mized) strategies is as follows. 
The column player chooses a distribution over the columns; that is, a vector 
p € [0,1]” with $; pi = 1. Similarly, the row player chooses a distribution 
q over the rows. The amount paid is the expectation of a;,; for 7 chosen from 
p and i chosen from q. If we think of p as a column vector and q as a row 
vector then this is equal to qAp. The min-max theorem says: 


min max qAp= max min qAp (7) 
pe[0,1]” qe[0,1]” qe[0,1]” pe[0,1]” 
2ip=1 Viqi=l 2iqu=1 Xipi=1 


The min-max theorem can be proven using the following result, known as 
Farkas’ Lemma:? if C and D are disjoint convex subsets of R”, then there 
is an m — 1 dimensional hyperplane that separates them. That is, there is 
a vector z and a number a such that for every x € C, (x,z) = 0, tizi <a 
and for every y € D, (y,z) > a. (A subset C C R™ is convex if 
whenever it contains a pair of points x,y, it contains the line segment 
{ax + (l—a)y:0<a< 1} that lies between them.) We ask you to prove 


Farkas’ Lemma in Exercise 2 but here is a “proof by picture” for the two 


dimensional case: - 


hyperplane 
Farkas’ Lemma implies the min-max theorem by noting that 
maxgminpqAp > c if and only if the convex set D = 
{Ap :p € [0,1]" J; pi =1} does not intersect with the convex set 
C= {x ER”: Viet zi < c} and using the Lemma to show that this 
implies the existence of a probability vector q such that (q, y) > c for every 
y € D (see Exercise 3). The Min-Max Theorem is equivalent to another 
well-known result called linear programming duality, that can also be proved 
using Farkas’ Lemma (see Exercise 4). 


17.4. ERROR CORRECTING CODES: THE INTUITIVE CONNECTION TO HARDNESS 
p17.8 (328) AMPLIFICATION 


17.4 Error correcting codes: the intuitive connection to hardness 
amplification 


Now we construct average-case hard functions using functions that are only worst-case hard. To do 
so, we desire a way to transform any function f to another function g such that if there is a small 
circuit that computes y approximately (i.e., correctly outputs g(x) for many x) then there is a small 
circuit that computes f at all points. Taking the contrapositive, we can conclude that if there is 
no small circuit that computes f then there is no small circuit that computes g approximately. 

Let us reason abstractly about how to go about the above task. 

View a function f : {0,1}” — {0,1} as its truth table, namely, as a string of length 2”, and 
view any circuit C for computing this function as a device that, given any index x € [2”], gives the 
z’th bit in this string. If the circuit only computes g on average” then this device may be thought 
of as only partially correct; it gives the right bit only for many indices x’s, but not all. Thus we 
need to show how to turn a partially correct string for g into a completely correct string for f. 
This is of course reminiscent of error correcting codes (ECC), but with a distinct twist involving 
computational efficiency of decoding, which we will call local decoding. 

The classical theory of ECC’s (invented by Shannon in 1949) concerns the following problem. 
We want to record some data x € [0,1)” on a compact disk to retrieve at a later date, but that 
compact disk might scratched and say 10% of its contents might be corrupted. The idea behind 
error correcting codes is to encode x using some redundancy so that such corruptions do not prevent 
us from recovering z. 

The naive idea of redundancy is to introduce repetitions but that does not work. For example 
suppose we repeat each bit three times, in other words encode z as the string y = 111111121913... TnTn Tn. 
But now if the first three coordinates of y are corrupted then we cannot recover x1, even if all other 
coordinates of y are intact. (Note that the first three coordinates take only a 1/n < 10% fraction 
of the entire string y.) Clearly, we need a smarter way. 


DEFINITION 17.8 (ERROR CORRECTING CODES) 

For x,y € 10,1)”, the fractional Hamming distance of x and y, denoted A(z, y), is 
equal to + |{i: x; 4 yi} |. 

For every 6 € [0,1], a function E : {0,1}" — {0,1} is an error correcting code 
(ECC) with distance ô, if for every x 4 y € {0,1}", A(E(x), E(y)) > ô. We call the 
set Im(E) = {E(x) : x € {0,1}"} the set of codewords of E. 


Suppose E : [0,1)” — {0,1} is an ECC of distance 6 > 0.2. Then the encoding x —> E(x) 
suffices for the CD storage problem (momentarily ignoring issues of computational efficiency). 
Indeed, if y is obtained by corrupting 0.1m coordinates of E(x), then A(y, E(x)) < 6/2 and by the 
triangle inequality A(y, E(x')) > 6/2 for every x’ # x. Thus, x is the unique string that satisfies 


2Many texts use the name Farkas’ Lemma only to denote a special case of the result stated in Note 17.7. Namely 
the result that there is a separating hyperplane between any disjoint sets C, D such that C is a single point and D 
is a set of the form {Ax : Via; > 0} for some matrix A. 
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Figure 17.1: In a d-distance error correcting code, A(E(x), E(x’)) > 6 for every x # x’. We can recover x from 
every string y satisfying A(y, E(x)) < 6/2 since the 6/2-radius ball around every codeword z = E(x) does not contain 
any other codeword. 


A(y, E(x)) < 6/2. (See Figure 17.1.) 

Of course, we still need to show that error correcting codes with minimum distance 0.2 actually 
exist. The following lemma shows this. It introduces H(0), the so-called entropy function, which 
lies strictly between 0 and 1 when 6 € (0,1). 


LEMMA 17.9 
For every ô < 1/2 and sufficiently large n, there exists a function E : {0,1}" > (0, 12 /4-H00) 
that is an error correcting code with distance ô, where H (ô) = 6log(1/6) + (1 — 6) log(1/(1 — 6)). 


PROOF: We simply choose the function E : {0,1}” — {0,1} at random for m = 2n/(1 — H(6)n. 
That is, we choose 2” random strings y1, ya,..., Yan and E will map the input x € {0,1}" (which 
we can identify with a number in [2”]) to the string yz. 

It suffices to show that the probability that for some i < j with i,j € [2"], A(yi, yj) < ô is less 
than 1. But for every string y;, the number of strings that are of distance at most 6 to it is (; im) 


which at most 0.99 - 2# (m for m sufficiently large (see Appendix A) and so for every j > i, the 
probability that y; falls in this ball is bounded by 0.99 - QH(d)m /2™. Since there are at most 22” 
such pairs 7,7, we only need to show that 


9H (Sym 
0.99 - a <1. 


which is indeed the case for our choice of m. M 


REMARK 17.10 

By a slightly more clever argument, we can get rid of the constant 2 above, and show that there 
exists such a code E : {0,1}" > {0, 1y 40H) (see Exercise 6). We do not know whether this is 
the smallest value of m possible. 


Why half? Lemma 17.9 only provides codes of distance 6 for 6 < 1/2 and you might wonder 
whether this is inherent or can we have codes of even greater distance. It turns out we can have 
codes of distance 1/2 but only if we allow m to be exponentially larger than n (i.e., m > 2/2). For 
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every ô > 1/2, if n is sufficiently large then there is no ECC E: {0,1}" — {0,1}” that has distance 
6, no matter how large m is. Both these bounds are explored in Exercise 7. 


The mere existence of an error correcting code is not sufficient for most applications: we need 
to actually be able to compute them. For this we need to show an explicit function E : {0,1}" > 
{0,1} that is an ECC satisfying the following properties: 


Efficient encoding There is a polynomial time algorithm to compute E(x) from z. 


Efficient decoding There is a polynomial time algorithm to compute x from every y such that 
A(y, E(x)) < p for some p. (For this to be possible, the number p must be less than 6/2, 
where ô is the distance of E.) 


There is a very rich and still ongoing body of work dedicated to this task, of which Section 17.5 
describes a few examples. 


17.4.1 Local decoding 


For use in hardness amplification, we need ECCs with more than just efficient encoding and decoding 
algorithms: we need local decoders, in other words, decoding algorithms whose running time is 
polylogarithmic. Let us see why. 

Recall that we are viewing a function from {0,1}" to {0,1} as a string of length 2”. To amplify 
its hardness, we take an ECC and map function f to its encoding E(f). To prove that this works, 
it suffices to show how to turn any circuit that correctly computes many bits of E(f) into a circuit 
that correctly computes all bits of f. This is formalized using a local decoder, which is a decoding 
algorithm that can compute any desired bit in the string for f using a small number of random 
queries in any string y that has high agreement with (in other words, low hamming distance to) 
E(f). Since we are interested in the circuits of size poly(n)— in other words, polylogarithmic in 2” 
—this must also be the running time of the local decoder. 


DEFINITION 17.12 (LOCAL DECODER) 

Let E : {0,1}” — {0,1}” be an ECC and let p and q be some numbers. A local decoder for E 
handling p errors is an algorithm L that, given random access to a string y such that A(y, E(x)) < p 
for some (unknown) x € {0,1}", and an index j € N, runs for polylog(m) time and outputs x; with 
probability at least 2/3. 


REMARK 17.13 
The constant 2/3 is arbitrary and can be replaced with any constant larger than 1/2, since the 
probability of getting a correct answer can be amplified by repetition. 

Notice, local decoding may be useful in applications of ECC’s that have nothing to do with 
hardness amplification. Even in context of CD storage, it seems nice if we do not to have to read 
the entire CD just to recover one bit of x. 


Using a local decoder, we can turn our intuition above of hardness amplification into a proof. 


17.4. ERROR CORRECTING CODES: THE INTUITIVE CONNECTION TO HARDNESS 


AMPLIFICATION 
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NOTE 17.11 (HIGH DIMENSIONAL GEOMETRY) 

While we are normally used to geometry in two or three dimensions, we can 
get some intuition on error correcting codes by considering the geometry of 
high dimensional spaces. Perhaps the strongest effect of high dimension is the 
following: compare the cube with all sides 1 and the ball of radius 1/4. In one 
dimension, the ratio between their areas is 1/(1/2) = 2, in two dimensions 
it is 1/(11/4?) = 16/7, while in three dimensions it is 1/(4/3711/4%) = 48/r. 
Note that as the number of dimension grows, this ratio grows exponentially 
in the number of dimensions. (Similarly for any two radii rı > r2 the volume 
of the m-dimension ball of radius rı is exponentially larger than the volume 
of the r2-radius ball.) 


0 1 14 1 o 1/4 3/4 1 O 1/4 3/4 1 
Ball volume=1/2 


B.V. = 1(1/4)?-3.14/16 B.V. =4/3m(1/4)? ~ 3.14/48 


This intuition lies behind the existence of an error correcting code with 
distance 1/4 mapping n bit strings into m = 5n bit strings. We can have gm/5 
codewords that are all of distance at least 1/4 from one another because, also 
in the Hamming distance, the volume of the radius 1/4 ball is exponentially 
smaller than the volume of the cube [0,1)”. Therefore, we can “pack” 2/5 
such balls within the cube. 
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length n string function on (0, 1)” = 
string of length 2" 


E(P) 
[YY corrupted E | YY voro | 


Figure 17.2: An ECC allows to map a string z to E(x) such as x can be reconstructed from a corrupted version of 
E(x). The idea is to treat a function f : (0,1)” —> {0,1} as a string in (0,1)?”, encode it using an ECC to a function 
f . Intuitively, f should be hard on the average case if f was hard on the worst case, since an algorithm to solve f 
with probability 1 — p could be transformed (using the ECC’s decoding algorithm) to an algorithm computing f on 
every input. 


THEOREM 17.14 

Suppose that there is an ECC with polynomial-time encoding algorithm and a local decoding 
algorithm handling p errors (where p is a constant independent of the input length). Suppose also 
that there is f € E with Had f)(n) > S(n) for some function S : N — N satisfying S(n) > n. Then, 
there exists e > 0 and g € E with H,.(g)(n) > S(en)* 


The proof of Theorem 17.14 follows essentially from the definition, and we will prove it for the 
case of a particular code later on in Theorem 17.24. 


17.5 Constructions of Error Correcting Codes 


We now describe some explicit functions that are error correcting codes, building up to the con- 
struction of an explicit ECC of constant distance with polynomial-time encoding and decoding. 
Section 17.6 describes local decoding algorithms for some of these codes. 


17.5.1 Walsh-Hadamard Code. 


For two strings x,y € {0,1}", define z © y to be the number >>;_, xiy; (mod 2). The Walsh- 
Hadamard code is the function WH : (0,1)” — {0,1}?" that maps a string x € (0,1)” into the 
string z € (0,1) where for every y € [0,1)”, the y!” coordinate of z is equal to x © y (we identify 
{0,1}” with [2”] in the obvious way). 


CLAIM 17.15 
The function WH is an error correcting code of distance 1/2. 
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| ZYcorrupted E( HA 


local 
decoder 


Figure 17.3: A local decoder gets access to a corrupted version of E(x) and an index i and computes from it 2; 
(with high probability). 


PROOF: First, note that WH is a linear function. By this we mean that if we take x + y to be the 
componentwise addition of x and y modulo 2, then WH(z + y) = WH(x) + WH(y). Now, for every 
x Æ y € {0,1}” we have that the number of 1’s in the string WH(x) + WH(y) = WH(a + y) is equal 
to the number of coordinates on which WH(x) and WH(y) differ. Thus, it suffices to show that for 
every z 4 0”, at least half of the coordinates in WH(z) are 1. Yet this follows from the random 
subsum principle (Claim A.5) that says that the probability for y Er (0,1)” that z © y = 1 is 
exactly 1/2. W 


17.5.2 Reed-Solomon Code 


The Walsh-Hadamard code has a serious drawback: its output size is exponential in the input size. 
By Lemma 17.9 we know that we can do much better (at least if we’re willing to tolerate a distance 
slightly smaller than 1/2). To get towards explicit codes with better output, we need to make a 
detour to codes with non-binary alphabet. 


DEFINITION 17.16 
For every set Y and x,y € E”, we define A(z, y) = + |{i : zi 4 yi}|. We say that E: =" — E” is 
an error correcting code with distance 9 over alphabet Y if for every x 4 y € Y”, A(E(x), E(y)) > ô. 


Allowing a larger alphabet makes the problem of constructing codes easier. For example, every 
ECC with distance 6 over the binary ({0,1}) alphabet automatically implies an ECC with the same 
distance over the alphabet {0,1,2,3}: just encode strings over {0,1,2,3} as strings over {0,1} in 
the obvious way. However, the other direction does not work: if we take an ECC over {0,1,2,3} 
and transform it into a code over {0,1} in the natural way, the distance might grow from 6 to 26 
(Exercise 8). 

The Reed-Solomon code is a construction of an error correcting code that can use as its alphabet 
any field F: 
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E,°E,:{0,1}"-->{0, 1} 


x 
E/:(0,1)P-->Em | 


TIE 
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Figure 17.4: If E1,E2 are ECC’s such that Er : {0,1}" — E” and Ez : o > {0,1}*, then the concatenated code 
E : {0,1}" — {0,1}”* maps « into the sequence of blocks E2(E1(x)1),...,£2(E1(x)m). 


DEFINITION 17.17 
Let F be a field and n,m numbers satisfying n < m < |F|. The Reed-Solomon code from F” to 


F™ is the function RS : F” > F™ that on input ag,...,@,_1 € F” outputs the string 20,...,Zm—1 
where 
n—-1 
a=) af; 
1=0 


and f; denotes the jt? element of F under some ordering. 


LEMMA 17.18 
The Reed-Solomon code RS : E” — F™ has distance 1 — nae 


PROOF: As in the case of Walsh-Hadamard code, the function RS is also linear in the sense that 
RS(a + b) = RS(a) + RS(b) (where addition is taken to be componentwise addition in F). Thus, as 
before we only need to show that for every a 4 0”, RS(a) has at most n coordinates that are zero. 
But this immediate from the fact that a nonzero n — 1 degree polynomial has at most n roots (see 
Appendix A). E 


17.5.3 Concatenated codes 


The Walsh-Hadamard code has the drawback of exponential-sized output and the Reed-Solomon 
code has the drawback of a non-binary alphabet. We now show we can combine them both to 
obtain a code without neither of these drawbacks: 


DEFINITION 17.19 

If RS is the Reed-Solomon code mapping F” to F™ (for some n,m,F) and WH is the Walsh- 
Hadamard code mapping {0,1}!°8! to (0, ine = {0,1}!"|, then the code WH o RS maps 
{0,1}"!°!*l to {0,1}! in the following way: 


1. View RS as a code from {0,1}"'°8!"! to F™ and WH as a code from F to {0,1}!"! using the 


canonical representation of elements in F as strings in {0,1}!°8!*!. 


2. For every input x € {0,1}"!°8 "|, WHoRS(z) is equal to WH(RS(z)1),..., WH(RS(x)m) where 
RS(x); denotes the 1% symbol of RS(z). 
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Note that the code WH o RS can be computed in time polynomial in n,m and |F|. We now 
analyze its distance: 


CLAIM 17.20 
Let 6; = 1 — n/m be the distance of RS and 62 = 1/2 be the distance of WH. Then WH o RS is an 
ECC of distance 6169. 


PROOF: Let z, y be two distinct strings in {0,1}!°8/"!", If we set a” = RS(2’) and y! = RS(y') then 
A(2’, y’) > 6,. If we let 2” (resp. y”) to be the binary string obtained by applying WH to each of 
these blocks, then whenever two blocks are distinct, the corresponding encoding will have distance 
92, and so ô(x”, y”) > 9102. M 


REMARK 17.21 
Because for every k € N, there exists a finite field |F| of size in |k, 2k] (e.g., take a prime in [k, 2k] or 
a power of two) we can use this construction to obtain, for every n, a polynomial-time computable 


ECC E: {0,1}" — {0, or of distance 0.4. 


Both Definition 17.19 and Lemma 17.20 easily generalize for codes other than Reed-Solomon 
and Hadamard. Thus, for every two ECC's Ej : {0,1}" > E” and Ez : E > {0,1}* their 
concatenation Ez o Ey is a code from {0,1}” to {0,1}"* that has distance at least 6152 where 6; 
(resp. 92) is the distance of Ej (resp. Ez), see Figure 17.6. In particular, using a different binary 
code than WH, it is known how to use concatenation to obtain a polynomial-time computable ECC 
E: 10,1)” — (0, 1)” of constant distance ô > 0 such that m = O(n). 


17.5.4 Reed-Muller Codes. 


Both the Walsh-Hadamard and and the Reed-Solomon code are special cases of the following family 
of codes known as Reed-Muller codes: 
DEFINITION 17.22 (REED-MULLER CODES) 
Let F be a finite field, and let Z, d be numbers with d < |F|. The Reed Muller code with parameters 
F, £,d is the function RM : A FE that maps every /-variable polynomial P over F of total 
degree d to the values of P on all the inputs in F°. 

That is, the input is a polynomial of the form 


_ X j i1 ni2 ig 
Ulises £e) = Cir... iet] Lo “Lo 
dy tiot...ptipg<l 


specified by the vector of ‘we coefficients {c;,,...;,} and the output is the sequence {g(a1,...,a¢)} 


for every £1,...,%¢ E F. 


Setting £ = 1 one obtains the Reed-Solomon code (for m = |F|), while setting d = 1 and 
F = GF(2) one obtains a slight variant of the Walsh-Hadamard code. (Le., the code that maps 
every x € {0,1}” into the 2-2” long string z such that for every y € [0,1)”,a € (0,1), zya = tOy+a 
(mod 2).) 

The Schwartz-Zippel Lemma (Lemma A.25 in Appendix A) shows that the Reed-Muller code 
is an ECC with distance 1 — d/|F|. Note that this implies the previously stated bounds for the 
Walsh-Hadamard and Reed-Solomon codes. 
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17.5.5 Decoding Reed-Solomon. 


To actually use an error correcting code to store and retrieve information, we need a way to 
efficiently decode a data x from its encoding E(x) even if E(x) has been corrupted in a fraction p 
of its coordinates. We now show this for the Reed-Solomon code, that treats x as a polynomial g, 
and outputs the values of this polynomial on m inputs. 

We know (see Theorem A.24 in Appendix A) that a univariate degree d polynomial can be 
interpolated from any d+ 1 values. Here we consider a robust version of this procedure, whereby 
we wish to recover the polynomial from m values of which pm are “faulty” or “noisy”. 

Let (a1, b1), (a2, b2),...,(@m, bm) be a sequence of (point, value) pairs. We say that a degree d 
polynomial g(x) describes this (a;,b;) if glai) = bi. 

We are interested in determining if there is a degree d polynomial g that describes (1 — p)m of 
the pairs. If 29m > d then this polynomial is unique (exercise). We desire to recover it, in other 
words, find a degree d polynomial g such that 


glas) = bi for at (1 — p)m least values of i. (8) 


The apparent difficulty is in identifying the noisy points; once those points are identified, we 
can recover the polynomial. 


Randomized interpolation: the case of p< 1/(d+1) 


If p is very small, say, p < 1/(2d) then we can actually use the standard interpolation technique: 
just select d+1 points at random from the set {(a;,};)} and use them to interpolate. By the union 
bound, with probability at least 1— p(d+1) > 0.4 all these points will be non-corrupted and so we 
will recover the correct polynomial. (Because the correct polynomial is unique, we can verify that 
we have obtained it, and if unsuccessful, try again.) 


Berlekamp-Welch Procedure: the case of p < (m — d)/(2m) 


The Berlekamp-Welch procedure works when the error rate p is bounded away from 1/2; specifically, 
p < (m — d)/(2m). For concreteness, assume m = 4d and p = 1/4. 


1. We claim that if the polynomial g exists then there is a degree 2d polynomial c(x) and a 
degree d nonzero polynomial e(x) such that 


c(a;) = b;e(a;) for all i. (9) 


The reason is that the desired e(x) can be any nonzero degree d polynomial whose roots are 
precisely the a;'s for which g(a;) Æ bi, and then just let c(x) = g(x)e(x). (Note that this is 
just an existence argument; we do not know g yet.)) 


2. Let c(x) = DN <q cit? and e(z) = Ney ez". The e;’s and c;’s are our unknowns, and these 
satisfy 4d linear equations given in (??), one for each a;. The number of unknowns is 3d + 2, 
and our existence argument in part 1 shows that the system is feasible. Solve it using Gaussian 
elimination to obtain a candidate c, e. 


17.6. LOCAL DECODING OF EXPLICIT CODES. p17.17 (337) 


3. Let c,e are any polynomials obtained in part 2. Since they satisfy (9) and b; = g(a;) for at 
least 3d values of i, we conclude that 


c(a;) = g(a;)e(a;) for at least 3d values of i. 


Hence c(x) — g(x)e(x) is a degree 2d polynomial that has at least 3d roots, and hence is 
identically zero. Hence e divides c and that in fact c(x) = g(x)e(zx). 


4. Divide c by e to recover g. 


17.5.6 Decoding concatenated codes. 


Decoding concatenated codes can be achieved through the natural algorithm. Recall that if Ej : 
{0,1}”" > E” and Ej : E > {0,1}* are two ECC's then Ez o Ey maps every string x € {0,1}” to 
the string Es(E¡(1)1):-- Es(E¡(2),). Suppose that we have a decoder for Ej (resp. E2) that can 
handle pı (resp. p2) errors. Then, we have a decoder for E20 E, that can handle p2p1 errors. The 
decoder, given a string y € (0, ¡pe composed of m blocks y;,...,Ym € (0, A first decodes each 
block y; to a symbol z; in X, and then uses the decoder of E to decode 21,...,Zm. The decoder 
can indeed handle p1p2 errors since if A(y, Ez o E¡(x1)) < pipa then at most pı of the blocks of y 
are of distance at least pa from the corresponding block of Es o E (x). 


17.6 Local Decoding of explicit codes. 


We now show local decoder algorithm (c.f. Definition 17.12) for several explicit codes. 


17.6.1 Local decoder for Walsh-Hadamard. 


The following is a two-query local decoder for the Walsh-Hadamard code that handles p errors for 
every p < 1/4. This fraction of errors we handle is best possible, as it can be easily shown that there 
cannot exist a local (or non-local) decoder for a binary code handling p errors for every p > 1/4. 


WALSH-HADAMARD LOCAL DECODER for p < 1/4: 


Input: j € [n], random access to a function f : (0,1)” — {0,1} such that Pry[g(y) ££ Oy] < p 
for some p < 1/4 and x € {0,1}". 


Output: A bit b € {0,1}. (Our goal: x; = b.) 


Operation: Let ef be the vector in {0,1}" that is equal to 0 in all the coordinates except for 
the jt” and equal to 1 on the jt” coordinate. The algorithm chooses y Er (0,1)” and 
outputs f(y) + f(y +e) (mod 2) (where y + ef denotes componentwise addition modulo 2, 
or equivalently, flipping the jt” coordinate of y). 


Analysis: Since both y and y+e/ are uniformly distributed (even though they are dependent), the 
union bound implies that with probability 1—2p, f(y) = r©y and f(y+e’) = xO(y+e’). But 
by the bilinearity of the operation ©, this implies that f(y)+ f(yte’?) = rOy+z2O(y+e’) = 
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Figure 17.5: Given access to a corrupted version of a polynomial P : F° — F, to compute P(x) we pass a random 
line Lz through x, and use Reed-Solomon decoding to recover the restriction of P to the line Le. 


2(10y)+20e =x0€el (mod 2). Yet, x © e) = zj and so with probability 1 — 2p, the 
algorithm outputs the right value. 


REMARK 17.23 

This algorithm can be modified to locally compute not just z; = xz © ef but in fact the value x © z 
for every z € {0,1}". Thus, we can use it to compute not just every bit of the original message x 
but also every bit of the uncorrupted codeword WH(x). This property is sometimes called the self 
correction property of the Walsh-Hadamard code. 


17.6.2 Local decoder for Reed-Muller 


We now show a local decoder for the Reed-Muller code. (Note that Definition 17.12 can be easily 
extended to the case of codes, such as Reed-Muller, that use non-binary alphabet.) It runs in 
time polynomial in £ and d, which, for an appropriate setting of the parameters, is polylogarithmic 
in the output length of the code. Convention: Recall that the input to a Reed-Muller code is 
an ¢-variable d-degree polynomial P over some field F. When we discussed the code before, we 
assumed that this polynomial is represented as the list of its coefficients. However, below it will be 
more convenient for us to assume that the polynomial is represented by a list of its values on its 
first (346) inputs according to some canonical ordering. Using standard interpolation, we still have 
a polynomial-time encoding algorithm even given this representation. Thus, it suffices to show an 
algorithm that, given access to a corrupted version of P, computes P(x) for every x € F“ 


REED-MULLER LOCAL DECODER for p < (1 — d/|F|)/4 — 1/|F]|. 


Input: A string x € F”, random access to a function f such that Pr,¢ge[P(x) 4 f(x)] < p, where 
P : Ff > F is an &variable degree-d polynomial. 


Output: y € F (Goal: y = P(x).) 


Operation: 1. Let Ly, be a random line passing through x. That is Lẹ = {x + ty : t € F} for a 
random y € F£. 
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Figure 17.6: To locally decode a concatenated code E2 o E we run the decoder for E; using the decoder for Ez. 
The crucial observation is that if y is within p1p2 distance to Ez o Eı(x) then at most a pı fraction of the blocks in 
y are of distance more than p2 the corresponding block in Ez o E, (x). 


2. Query f on all the |F| points of L, to obtain a set of points {(t, f(x + ty))} for every 
ter. 

3. Run the Reed-Solomon decoding algorithm to obtain the univariate polynomial Q : F = 
F such that Q(t) = f(x + ty) for the largest number of t’s (see Figure 17.5).* 


4. Output Q(0). 


Analysis: For every d-degree ¢-variable polynomial P, the univariate polynomial Q(t) = P(x+ty) 
has degree at most d. Thus, to show that the Reed-Solomon decoding works, it suffices to 
show that with probability at least 1/2, the number of points on z € Lg for which f(z) 4 P(z) 
is less than (1 — d/|F|)/2. Yet, for every t 4 0, the point x + ty is uniformly distributed 
(independently of x), and so the expected number of points on Ly for which f and P differ 
is at most p|F| + 1. By Markov inequality, the probability that there will be more than 
2p|F| +2 < (1 — d/|F|)|F|/2 such points is at most 1/2 and hence Reed-Solomon decoding will 
be successful with probability 1/2. In this case, we obtain the correct polynomial q that is the 
restriction of Q to the line Ly and hence q(0) = P(x). 


17.6.3 Local decoding of concatenated codes. 


Given two locally decodable ECC’s Ej and E2, we can locally decode their concatenation Ej o Ea 
by the natural algorithm. Namely, we run the decoder for Ej, but answer its queries using the 
decoder for Ez (see Figure 17.6). 


LOCAL DECODER FOR CONCATENATED CODE: p< p1p2 


The code: If E, : {0,1}" + E” and Ey : E > {0,1}* are codes with decoders of qı (resp. q2) 
queries with respect to pı (resp. p2) errors, let E = E20 E; be the concatenated code mapping 
{0,1}” to {0,1}. 


3If p is sufficiently small, (e.g., p < 1/(10d)), then we can use the simpler randomized Reed-Solomon decoding 
procedure described in Section 17.5.5. 
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Input: An index i € [n], random access to a string y € (0,1)%” such that A(y, E1 o E2(x)) < pips 
for some x € {0,1}”. 


Output: be (0,1)” (Goal: b = x;) 


Operation: Simulate the actions of the decoder for E1, whenever the decoder needs access to the 
jt symbol of E(x), use the decoder of Es with O(q2 log qı log |X|) queries applied to the j” 
block of y to recover all the bits of this symbol with probability at least 1 — 1/(2q1). 


Analysis: The crucial observation is that at most a pı fraction of the length k blocks in y can 
be of distance more than pz from the corresponding blocks in Ez o E,(x). Therefore, with 
probability at least 0.9, all our q, answers to the decoder of Ej are consistent with the answer 
it would receive when accessing a string that is of distance at most pı from a codeword of E). 


17.6.4 Putting it all together. 


We now have the ingredients to prove our second main theorem of this chapter: transformation of 
a hard-on-the-worst-case function into a function that is “mildly” hard on the average case. 


THEOREM 17.24 (WORST-CASE HARDNESS TO MILD HARDNESS) 

Let S: N — N and f € E such that Had f)(n) > S(n) for every n. Then there exists 
a function y € E and a constant c > 0 such that H2.°%g)(n) > S(n/c)/n* for every 
sufficiently large n. 


PROOF: For every n, we treat the restriction of f to {0,1}” as a string f’ € {0,1} where N = 2". 


We then encode this string f’ using a suitable error correcting code E : {0,1}. — (0, g for 
some constant C > 1. We will define the function g on every input x € {0, 1y” to output the a” 
coordinate of E(f”).* For the function g to satisfy the conclusion of the theorem, all we need is for 
the code E to satisfy the following properties: 


1. For every x € {0,1}%, E(x) can be computed in poly(N) time. 


2. There is a local decoding algorithm for E that uses polylog(N) running time and queries and 
can handle a 0.01 fraction of errors. 


But this can be achieved using a concatenation of a Walsh-Hadamard code with a Reed-Muller 
code of appropriate parameters: 


1. Let RM denote the Reed-Muller code with the following parameters: 


e The field F is of size log? N. 
e The number of variables £Z is equal to log N/ log log N. 


“By padding with zeros as necessary, we can assume that all the inputs to g are of length that is a multiple of C. 
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e The degree is equal to log? N. 


RM takes an input of length at least (oy > N (and so using padding we can assume its input 
is {0,1}"). Its output is of size [F|* < poly(n). Its distance is at least 1 — 1/log N. 


2. Let WH denote the Walsh-Hadamard code from (0,18% = (0, 1]PeloN to (0, 1}E! = 
{0, 1 pos? N 


Our code will be WHoRM. Combining the local decoders for Walsh-Hadamard and Reed-Muller 
we get the desired result. M 


Combining Theorem 17.24 with Yao’s XOR Lemma (Theorem 17.2), we get the following corol- 
lary: 


COROLLARY 17.25 

Let S : N > N and f € E with H,.{f)(n) > S(n) for every n. Then, there exists an S(v£)“- 
pseudorandom generator for some constant e > 0. 

PROOF: By Theorem 17.24, under this assumption there exists a function g € E with H?%q)(n) > 
S'(n) = S(n)/poly(n), where we can assume S'(n) > \/S(n) for sufficiently large n (otherwise S 
is polynomial and the theorem is trivial). Consider the function g®* where k = clog S'(n) for a 
sufficiently small constant c. By Yao’s XOR Lemma, on inputs of length kn, it cannot be computed 
with probability better than 1/2 + 27*5'()/1000 by circuits of size S’(n). Since S(n) < 2”, kn < yn, 
and hence we get that H,,(g®*) > gc/2000 y 


As already mentioned, this implies the following corollaries: 


1. If there exists f € E such that H,,(f) > ar") then BPP C QuasiP. 


2. If there exists f € E such that Hurd f) > n“() then BPP C SUBEXP. 


However, Corollary 17.25 is still not sufficient to show that BPP = P under any assumption on 
the worst-case hardness of some function in E. It only yields an S (V0)°-pseudorandom generator, 
while what we need is an $(2())°-pseudorandom generator. 


17.7 List decoding 


Our approach to obtain stronger worst-case to average-case reduction will be to bypass the XOR 
Lemma, and use error correcting codes to get directly from worst-case hardness to a function that 
is hard to compute with probability slightly better than 1/2. However, this idea seems to run into 
a fundamental difficulty: if f is worst-case hard, then it seems hard to argue that the encoding 
of f, under any error correcting code is hard to compute with probability 0.6. The reason is that 
any error-correcting code has to have distant at most 1/2, which implies that there is no decoding 
algorithm that can recover x from E(x) if the latter was corrupted in more than a 1/4 of its locations. 
Indeed, in this case there is not necessarily a unique codeword closest to the corrupted word. For 
example, if E(x) and E(x’) are two codewords of distance 1/2, let y be the string that is equal to 
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E(x) on the first half of the coordinates and equal to E(x’) on the second half. Given y, how can 
a decoding algorithm know whether to return x or x’? 

This seems like a real obstacle, and indeed was considered as such in many contexts where 
ECC’s were used, until the realization of the importance of the following insight: “If y is obtained 
by corrupting E(x) in, say, a 0.4 fraction of the coordinates (where E is some ECC with good 
enough distance) then, while there may be more than one codeword within distance 0.4 to y, there 
can not be too many such codewords.” 


THEOREM 17.26 (JOHNSON BOUND) 
If E : {0,1}" — {0,1} is an ECC with distance at least 1/2 — e, then for every x € {0,1}, and 
ô > ye, there exist at most 1/(29?) vectors y1,...,ye such that A(z, yi) < 1/2 — ô for every i € [4]. 


PROOF: Suppose that x,y1,..., ye satisfy this condition, and define £ vectors z1,..., ze in R” as 
follows: for every i € [£] and k € [m], set 2; y to equal +1 if yk = x, and set it to equal —1 otherwise. 
Under our assumptions, for every i € [4], 


m 
> Zi,k 2 26m, (10) 
k=1 


since z; agrees with x on an 1/2 + 6 fraction of its coordinates. Also, for every ¿4 j € |, 
m 
== 5S Aine < 2em < 28m (11) 
k=1 
since E is a code of distance at least 1/2 — e. We will show that (10) and (11) together imply that 
£ < 1/(28?). 
Indeed, set w = e zi. On one hand, by (11) 


£ 
(w, w) = SN las zi) + Do zj) < lm + (25m. 
i=1 Aj 


On the other hand, by (10), > =>; j Zij 2 26mé and hence 
(w,w) > | wel?/m > 48' mé”, 
k 


since for every c, the vector w € R” with minimal two-norm satisfying >, wz = c is the uniform 
vector (c/m,e/m,...,c/m). Thus 462mé? < 4m + 22282m, implying that £ < 1/(29?). m 


17.7.1 List decoding the Reed-Solomon code 


In many contexts, obtaining a list of candidate messages from a corrupted codeword can be just as 
good as unique decoding. For example, we may have some outside information on which messages 
are likely to appear, allowing us to know which of the messages in the list is the correct one. 
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However, to take advantage of this we need an efficient algorithm that computes this list. Such an 
algorithm was discovered in 1996 by Sudan for the popular and important Reed-Solomon code. It 
can recover a polynomial size list of candidate codewords given a Reed-Solomon codeword that was 
corrupted in up to a 1 — 2\/d/|F| fraction of the coordinates. Note that this tends to 1 as |F|/d 
grows, whereas the Berlekamp-Welch unique decoding algorithm of Section 17.5.5 gets “stuck” 
when the fraction of errors surpasses 1/2. 

On input a set of data points {(a;,b;)}%, in F?, Sudan’s algorithm returns all degree d poly- 
nomials g such that the number of 2's for which g(a;) = b; is at least 2\/d/|F|m. It relies on the 
following observation: 


LEMMA 17.27 

For every set of m data pairs (a1, b1),...,(@m, bm), there is a bivariate polynomial Q(z, x) of degree 
at most [y/m| +1 in z and x such that Q(b;,a;) = 0 for each i = 1,...,m. Furthermore, there is 
a polynomial-time algorithm to construct such a Q. 


PROOF: Let k = [ym] +1. Then the unknown bivariate polynomial Q = Ss Sy QijzřxÍ has 
(k + 1)? coefficients and these coefficients are required to satisfy m linear equations of the form: 

k k 
S y aia) fre Lim 


i=0 j=0 


Note that the a,’s, b¿'s are known and so we can write down these equations. 
Since the system is homogeneous and the number of unknowns exceeds the number of con- 
straints, it has a nonzero solution. Furthermore this solution can be found in polynomial time. M 


LEMMA 17.28 
Let d be any integer and k > (d+ 1)([vm] + 1). If p(x) is a degree d polynomial that describes k 
of the data pairs, then z — p(x) divides the bivariate polynomial Q(z, x) described in Lemma 17.27. 


PROOF: By construction, Q(b:,a,;) = 0 for every data pair (a;,b;). If p(x) describes this data 
pair, then Q(p(az),a:) = 0. We conclude that the univariate polynomial Q(p(x),z) has at least k 
roots, whereas its degree is d([yn| + 1) < k. Hence Q(p(x),x) = 0. By the division algorithm 
for polynomials, Q(p(x),x) is exactly the remainder when Q(z,x) is divided by (z — p(x)). We 
conclude that z — p(x) divides Q(z, x). Wm 


Now it is straightforward to describe Sudan’s list decoding algorithm. First, find Q(z, x) by 
the algorithm of Lemma 17.27. Then, factor it using a standard algorithm for bivariate factoring 
(see [VG99]). For every factor of the form (z — p(x)), check by direct substitution whether or not 
p(x) describes 2,/d/|F|m data pairs. Output all such polynomials. 


17.8 Local list decoding: getting to BPP = P. 


Analogously to Section 17.4.1, to actually use list decoding for hardness amplification, we need to 
provide local list decoding algorithms for the codes we use. Fortunately, such algorithms are known 
for the Walsh-Hadamard code, the Reed-Muller code, and their concatenation. 
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DEFINITION 17.29 (LOCAL LIST DECODER) 

Let E : [0,1)” — {0,1} be an ECC and let p > 0 and q be some numbers. An algorithm L is 
called a local list decoder for E handling p errors, if for every x € {0,1}" and y € {0,1}” satisfying 
A(E(z),y) < p, there exists a number iy € [poly(n/e)] such that for every j € [m], on inputs ig, 7 
and with random access to y, L runs for poly(log(m)/e) time and outputs x; with probability at 
least 2/3. 


REMARK 17.30 
One can think of the number ig as the index of x in the list of poly(n/e) candidate messages output 
by L. Definition 17.29 can be easily generalized to codes with non-binary alphabet. 


17.8.1 Local list decoding of the Walsh-Hadamard code. 


It turns out we already encountered a local list decoder for the Walsh-Hadamard code: the proof 
of the Goldreich-Levin Theorem (Theorem 10.14) provided an an algorithm that given access to 
a “black box” that computes the function y + x © y (for x,y € ([0,1)”) with probability 1/2 + e, 
computes a list of values 71,...  Epoly(n/e) such that £i = x for some ig. In the context of that 
theorem, we could find the right value of x from that list by checking it against the value f(x) 
(where f is a one-way permutation). This is a good example for how once we have a list decoding 


algorithm, we can use outside information to narrow the list down. 


17.8.2 Local list decoding of the Reed-Muller code 


We now present an algorithm for local list decoding of the Reed-Muller code. Recall that the 
codeword of this code is the list of evaluations of a d-degree ¢-variable polynomial P : Ff — F. The 
local decoder for Reed-Muller gets random access to a corrupted version of P and two inputs: an 
index i and x € F°. Below we describe such a decoder that runs in poly(d, £, |F|) and outputs P(x) 
with probability at least 0.9 assuming that 7 is equal to the “right” index ip. Note: To be a valid 
local list decoder, given the index ig, the algorithm should output P(x) with high probability for 
every x € F*. The algorithm described below is only guaranteed to output the right value for most 
(i.e., a 0.9 fraction) of the x’s in F*. We transform this algorithm to a valid local list decoder by 
combining it with the Reed-Muller local decoder described in Section 17.6.2. 


REED-MULLER LOCAL List DECODER for p < 1 — 10\/d/|F| 


Inputs: e Random access to a function f such that Pr epe|P(£) = f(x)] > 10\/d/|F| where 
P : Ff > F is an /-variable d-degree polynomial. We assume |F| > d* and that both 
d > 1000. (This can always be ensured in our applications.) 


e An index ig € [|F|*+"] which we interpret as a pair (xo, yo) with zo € Ff, yo € F, 
e A string x € F*. 


Output: y € F (For some pair (x0, yo), it should hold that P(x) = y with probability at least 0.9 
over the algorithm’s coins and x chosen at random from F*.) 
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Operation: 1. Let Lr zọ be a random degree 3 curve passing through x, xo. That is, we find a 
random degree 3 univariate polynomial q : F > F* such that q(0) = x and q(r) = zo for 
some random r € F. (See Figure 17.7.) 


2. Query f on all the |F| points of Lz.) to obtain the set S of the |F| pairs {(t, f(¢(t)) : 
te F)}. 


3. Run Sudan’s Reed-Solomon list decoding algorithm to obtain a list g1, .. . , gx of all degree 
3d polynomials that have at least 8,/d|F| agreement with the pairs in S. 


4. If there is a unique i such that g;(r) = yo then output g;(0). Otherwise, halt without 
outputting anything. 


Figure 17.7: Given access to a corrupted version of a polynomial P : F° — F and some index (Zo, yo), to compute 
P(x) we pass a random degree-3 curve Ly through x and xo, and use Reed-Solomon list decoding to recover a list 
of candidates for the restriction of P to the curve Lz, sọ. If only one candidate satisfies that its value on xo is yo, 
then we use this candidate to compute P(x). 


We will show that for every f : Ff — F that agrees with an /-variable degree d polynomial 
on a 10,/d/|F] fraction of its input, and every x € F*, if xy is chosen at random from F° and 
yo = P(xo), then with probability at least 0.9 (over the choice of xo and the algorithm’s coins) the 
above decoder will output P(x). By a standard averaging argument, this implies that there exist 
a pair (xo, yo) such that given this pair, the algorithm outputs P(x) for a 0.9 fraction of the x’s in 
Re. 

Let x € Fl, if xo is chosen randomly in F’ and yo = P(xo) then the following 

For every x € F*, the following fictitious algorithm can be easily seen to have an identical output 
to the output of our decoder on the inputs 2, a random xp Er F° and yo = P(ao): 


1. Choose a random degree 3 curve L that passes through z. That is, L = {q(t) : t € F} where 
q : F — F* is a random degree 3 polynomial satisfying q(0) = zx. 


2. Obtain the list g1,...,9m of all univariate polynomials over F such that for every i, there are 
at least 6,/d|F| values of t such that g;(t) = f(q(t)). 


3. Choose a random r € F. Assume that you are given the value yo = P(q(r)). 
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4. If there exists a unique 7 such that g;(r) = yo then output g;(0). Otherwise, halt without an 
input. 


Yet, this fictitious algorithm will output P(x) with probability at least 0.9. Indeed, since all 
the points other than x on a random degree 3 curve passing through x are pairwise independent, 
Chebyshev’s inequality implies that with probability at least 0.99, the function f will agree with 
the polynomial P on at least 8,/d|F| points on this curve (this uses the fact that \/d/|F| is smaller 
than 1074). Thus the list g1,..., gm we obtain in Step 2 contains the polynomial g : F — F defined 
as g(t) = P(q(t)). We leave it as Exercise 9 to show that there can not be more than \/|F|/4d 
polynomials in this list. Since two 3d-degree polynomials can agree on at most 3d+ 1 points, with 


(3d+1) V/F] 


probability at least CHD ES < 0.01, if we choose a random r € F, then g(r) 4 g;(r) for every 
gi + g in this list. Thus, with this probability, we will identify the polynomial g and output the 
value g(0) = P(x). E 


17.8.3 Local list decoding of concatenated codes. 


If E, : {0,1}" — E” and Ey : E > {0,1}* are two codes that are locally list decodable then so 
is the concatenated code Ez o E; : 10,1)” > {0,1}"*. As in Section 17.6.3, the idea is to simply 
run the local decoder for Ej while answering its queries using the decoder of Es. More concretely, 
assume that the decoder for Ej takes an index in the set I4, uses qı queries, and can handle 1 — e, 
errors, and that I2, q2 and ez are defined analogously. Our decoder for Es o Ej will take a pair 
of indices 11 € Jy and ig € I2 and run the decoder for Ej with the index 21, and whenever this 
decoder makes a query answer it using the decoder E with the index iz. (See Section 17.6.3.) We 
claim that this decoder can handle 1/2 — €,€2|I2| number of errors. Indeed, if y agrees with some 
codeword E20 Fj (x) on an €1€2|J2| fraction of the coordinates then there are €;|J2| blocks on which 
it has at least 1/2 + ez agreement with the blocks this codeword. Thus, by an averaging argument, 
there exists an index ¿2 such that given iz, the output of the Ez decoder agrees with E(x) on e 
symbols, implying that there exists an index i; such that given (71,72) and every coordinate j, the 
combined decoder will output x; with high probability. 


17.8.4 Putting it all together. 


As promised, we can use local list decoding to transform a function that is merely worst-case hard 
into a function that cannot be computed with probability significantly better than 1/2: 


THEOREM 17.31 (WORST-CASE HARDNESS TO STRONG HARDNESS) 

Let S: N — N and f € E such that H,.{f)(n) > S(n) for every n. Then there exists 
a function g € E and a constant c > 0 such that H,¿g)(n) > S(n/c)\/° for every 
sufficiently large n. 


PROOF SKETCH: As in Section 17.6.4, for every n, we treat the restriction of f to {0,1}”" as a 
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string f’ € {0,1}. where N = 2” and encode it using the concatenation of a Reed-Muller code 
with the Walsh-Hadamard code. For the Reed-Muller code we use the following parameters: 


e The field F is of size S(n)1/10, 5 


e The degree d is of size log? N. 


e The number of variables £ is 2log N/ log S(n). 


The function g is obtained by applying this encoding to f. Given a circuit of size S (nm) 100 
that computes g with probability better than 1/2 + 1/9 (n)! 50 we will be able to transform it, in 
S (n)20) time, to a circuit computing f perfectly. We hardwire the index i to this circuit as part 


of its description. Mi 


WHAT HAVE WE LEARNED? 


Yao’s XOR Lemma allows to amplify hardness by transforming a Boolean 
function with only mild hardness (cannot be computed with say 0.99 success) 
into a Boolean function with strong hardness (cannot be computed with 0.51 
success). 


An error correcting code is a function that maps every two strings into a pair 
of strings that differ on many of their coordinates. An error correcting code 
with a local decoding algorithm can be used to transform a function hard in 
the worst-case into a function that is mildly hard on the average case. 


A code over the binary alphabet can have distance at most 1/2. A code with 
distance 6 can be uniquely decoded up to 6/2 errors. List decoding allows to 
a decoder to handle almost a 6 fraction of errors, at the expense of returning 
not a single message but a short list of candidate messages. 


We can transform a function that is merely hard in the worst case to a function 
that is strongly hard in the average case using the notion of local list decoding 
of error correcting codes. 


Chapter notes and history 


MANY ATTRIBUTIONS STILL MISSING. 


Impagliazzo and Wigderson [IWO01] were the first to prove that BPP = P if there exists 
f € E such that H,(f) > 201) using a derandomized version of Yao’s XOR Lemma. However, 


We assume here that S(n) > log N'°° and that it can be computed in 2°“) time. These assumptions can 
be removed by slightly complicating the construction (namely, executing it while guessing that S(n) = 2%, and 


concatenating all the results.) 
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the presentation here follows Sudan, Trevisan, and Vadhan [STV], who were the first to point the 
connection between local list decoding and hardness amplification, and gave (a variant of) the Reed- 
Muller local list decoding algorithm described in Section 17.8. They also showed a different approach 
to achieve the same result, by first showing that the NW generator and a mildly hard function can 
be used to obtain from a short random seed a distribution that has high pseudoentropy, which is 
then converted to a pseudorandom distribution via a randomness extractor (see Chapter 16). 

The question raised in Problem 5 is treated in O’Donnell [O’D04], where a hardness amplification 
lemma is given for NP. For a sharper result, see Healy, Vadhan, and Viola [HV V04]. 


Exercises 


81 Let X;,..., Xn be independent random variables such that X; is equal to 1 with probability 
1 — ô and equal to 0 with probability ô. Let X = Ka Xi (mod 2). Prove that Pr[X = 1] = 
1/2+ (1 —26)*. 


*"suoryeyoodxa Mey} Jo yonpoid əy} st 
SO[QBLIVA wopueLr juepuedaput jo yonpord e jo voryejoodxo əy} yey} 


ye] oqy osn wL A 1] = A Pue (1) = 4 opa: H 


$2 Prove Farkas’ Lemma: if C, D C R™ are two convex sets then there exists a vector z € R™ 
and a number a € R such that 


‘qd > 4 pue p Ð X 10] A — X W10} OY JO 10799A 
38991078 ƏY} 9Q 0} Z 9387 ULI NOÁ osed SIYY UT ‘Gq Ð A pue J>X 
£ioa9 107 9 Z *||A—x||  < > 9ULOS 10} ey} SUBSE YI YM ‘poyeredos 
23 918 q Pue O yey} eses oy} ul sty} Fuod Aq 91898 :JUIH 


83 Prove the Min-Max Theorem (see Note 17.7) using Farkas’ Lemma. 


$4 Prove the duality theorem for linear programming using Farkas’ Lemma. That is, prove that 
for every m x n matrix A, and vectors c € R”, b € R”, 


max (x,c)= min (y,b 
A i ) snes í ) 
x= Aly> 
x>0 yo 


where At denotes the transpose of A and for two vectors u,v we say that u > v if u; > vi 
for every i. 


$5 Suppose we know that NP contains a function that is weakly hard for all polynomial-size 
circuits. Can we use the XOR Lemma to infer the existence of a strongly hard function in 
NP? Why or why not? 
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$6 


87 


$8 


89 


$10 


For every 6 < 1/2 and sufficiently large n, prove that there exists a function E : {0,1}" = 
{0,1}"/4-#©)) that is an error correcting code with distance 6, where H(6) = dlog(1/6) + 
(1 — 6) log(1/(1 — 6)). 


LINIS 393 NOA ppm Us AA “S9UO 
SNOLA9IH 09 9 IDULISIP UPA SI YY] pIOMIPO) Y ZUTPPR I9A9U “DUO 
Áq əuo ¿7 JO SpIOMIDPOI əy} YeTes 0} “A397e.19s ÁpaoI3 e 98/]:JUTH 


Show that for every E : {0,1}” — {0,1}” that is an error correcting code of distance 1/2, 
2” < 10,/n. Show if E is an error correcting code of distance ô > 1/2, then 2” < 10/(0 — 1/2). 


Let E : (0,1)” > {0,1}™ be a d-distance ECC. Transform E to a code E’ : {0,1,2,3}"/? > 
(0, Lose 2 in the obvious way. Show that E” has distance 6. Show that the opposite 
direction is not true: show an example of a 6-distance ECC E’ : {0,1, 2, 3y? — {0,1,2, aye 
such that the corresponding binary code has distance 20. 


Let f:F — F be any function. Suppose integer d > 0 and number e satisfy € > 2 tr . Prove 
that there are at most 2/e degree d polynomials that agree with f on at least an e fraction of 
its coordinates. 
oy ‘9 = ZS U IS oro zç syurod 
jo uoryoea |q|/p — > ut f soqiiosep peruouAjod puovəs ayy ‘To Aes 
syurod jo worry > ue ut f soquiosep peruouAjod 9814 oy, JUH 


(Linear codes) We say that an ECC E : {0,1}” — {0,1}" is linear if for every x, 2’ € {0,1}”, 
E(1+11) = E(x) + E(x”) where + denotes componentwise addition modulo 2. A linear ECC 
E can be described by an m x n matrix A such that (thinking of x as a column vector) 
E(x) = Az for every x € {0,1}". 


(a) Prove that the distance of a linear ECC E is equal to the minimum over all nonzero 
x € 10,1)” of the fraction of 1’s in E(x). 
(b) Prove that for every ô > 0, there exists a linear ECC E : 10,1)” > (0,1 1/4-H0) 
with distance 6, where H(0) = ô log(1/6) + (1 — 6) log(1/(1 — 6))z 
“XLIYeUr 
WIOpURI Y 10] SPfOY sty} moys - poyjour onsyqeqord oy 98/): PUTH 
(c) Prove that for some 6 > 0 there is an ECC E: {0,1}" — {0, 1yPoly(n) of distance 6 with 
polynomial-time encoding and decoding mechanisms. (You need to know about the field 
GF(2*) to solve this, see Appendix A.) 
‘opoo PIRUIRPeH-USTEM oy} 
YA (yZ) 40 190 UOUIO[OS-PO99Y JO UOTFRU9JBIUO) oY} ON FUH 
(d) We say that a linear code E : {0,1}” — {0,1}” is e-biased if for every non-zero x € 
[0, 1)”, the fraction of 1's in E(x) is between 1/2—e and 1/2+e. Prove that for every e > 0, 
there exists an e-biased linear code E : (0,1)” — (0, 1yPoly(n/ © with a polynomial-time 
encoding algorithm. 
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Chapter 18 


PCP and Hardness of Approximation 


“ ..most problem reductions do not create or preserve such gaps...To create such a 
gap in the generic reduction (cf. Cook)...also seems doubtful. The intuitive reason 
is that computation is an inherently unstable, non-robust mathematical object, in the 
the sense that it can be turned from non-accepting to accepting by changes that would 
be insignificant in any reasonable metric.” 

Papadimitriou and Yannakakis, 1991 [PY91] 


The PCP Theorem provides an interesting new characterization for NP, as the set of languages 
that have a “locally testable” membership proof. It is reminiscent of —and was motivated by— 
results such as IP =PSPACE. Its essence is the following: 

Suppose somebody wants to convince you that a Boolean formula is satisfiable. He could present 
the usual certificate, namely, a satisfying assignment, which you could then check by substituting 
back into the formula. However, doing this requires reading the entire certificate. The PCP 
Theorem shows an interesting alternative: this person can easily rewrite his certificate so you 
can verify it by probabilistically selecting a constant number of locations—as low as 3 bits— to 
examine in it. Furthermore, this probabilistic verification has the following properties: (1) A 
correct certificate will never fail to convince you (that is, no choice of your random coins will make 
you reject it) and (2) If the formula is unsatisfiable, then you are guaranteed to reject every claimed 
certificate with high probability. 

Of course, since Boolean satisfiability is NP-complete, every other NP language can be deter- 
ministically and efficiently reduced to it. Thus the PCP Theorem applies to every NP language. 
We mention one counterintuitive consequence. Let A be any one of the usual axiomatic systems of 
mathematics for which proofs can be verified by a deterministic TM in time that is polynomial in 
the length of the proof. Recall the following language is in NP: 


L = {(y,1") : p has a proof in A of length <n}. 


The PCP Theorem asserts that L has probabilistically checkable certificates. Such certificate 
can be viewed as an alternative notion of “proof” for mathematical statements that is just as valid 
as the usual notion. However, unlike standard mathematical proofs, where every line of the proof 


p18.1 (351) 
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has to be checked to verify its validity, this new notion guarantees that proofs are probabilistically 
checkable by examining only a constant number of bits in them!. 

This new, “robust” notion of certificate/proof has an important consequence: it implies that 
many optimization problems are NP-hard not only to solve exactly but even to approximate. As 
mentioned in Chapter 2, the P versus NP question is practically important —as opposed to “just” 
philosophically important— because thousands of real-life combinatorial optimization problems are 
NP-hard. By showing that even computing approximate solutions to many of these problems is 
NP-hard, the PCP Theorem extends the practical importance of the theory of NP-completeness, 
as well as its philosophical significance. 

This seemingly mysterious connection between the PCP Theorem —which concerns probabilis- 
tic checking of certificates— and the NP-hardness of computing approximate solutions is actually 
quite straightforward. All NP-hardness results ultimately derive from the Cook-Levin theorem 
(Section 2.3), which expresses accepting computations of a nondeterministic Turing Machine with 
satisfying assignments to a Boolean formula. Unfortunately, the standard representations of com- 
putation are quite nonrobust, meaning that they can be incorrect if even one bit is incorrect (see 
the quote at the start of this chapter). The PCP Theorem, by giving a robust representation of 
the certificate for NP languages, allow new types of reductions; see Section 18.2.3. 

Below, we use the term “PCP Theorems” for the body of other results of a similar nature to 
the PCP Theorem that found numerous applications in complexity theory. Some important ones 
appear in the next Chapter, including one that improves the PCP Theorem so that verification is 
possible by reading only 3 bits in the proof! 


18.1 PCP and Locally Testable Proofs 


According to our usual definition, language L is in NP if there is a poly-time Turing machine V 
(“verifier”) that, given input z, checks certificates (or membership proofs) to the effect that x € L. 
This means, 


ee L> Jr s.t. Ve) =1 
c¢L>Vn V(r) =0, 


where V” denotes “a verifier with access to certificate 7”. 

The class PCP (short for “Probabilistically Checkable Proofs” ) is a generalization of this notion, 
with the following changes. First, the verifier is probabilistic. Second, the verifier has random access 
to the proof string II. This means that each bit of the proof string can be independently queried 
by the verifier via a special address tape: if the verifier desires say the ith bit in the proof string, 
it writes į on the address tape and then receives the bit 7[1].2 (This is reminiscent of oracle TMs 
introduced in Chapter 3.) The definition of PCP treats queries to the proof as a precious resource, 
to be used sparingly. Note also that since the address size is logarithmic in the proof size, this model 
in principle allows a polynomial-time verifier to check membership proofs of exponential size. 


‘One newspaper article about the discovery of the PCP Theorem carried the headline “New shortcut found for 
long math proofs!” 

2Though widely used, the term “random access” is misleading since it doesn’t involve any notion of randomness 
per se. “Indexed access” would be more accurate. 
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Verifiers can be adaptive or nonadaptive. A nonadaptive verifier selects its queries based only 
on its input and random tape, whereas an adaptive verifier can in addition rely upon bits it has 
already queried in 7 to select its next queries. We restrict verifiers to be nonadaptive, since most 
PCP Theorems can be proved using nonadaptive verifiers. (But Exercise 3 explores the power of 
adaptive queries.) 


proof: 7 | | | | | | 


Verifier 
Input: x in {0,1}" 
r(n) coins 


Figure 18.1: A PCP verifier for a language L gets an input x and random access to a string m. If x € L then there 
exists a string 7 that makes the verifier accepts, while if x ¢ L then the verifier rejects every proof m with probability 
at least 1/2. 


DEFINITION 18.1 ((r, q)-VERIFIER) 
Let L be a language and q,r : N > N. We say that L has an (r(n), q(n))-verifier if there's a 
polynomial-time probabilistic algorithm V satisfying: 


Efficiency: On input a string x € {0,1}" and given random access to a string m € {0,1}* (which 
we call the proof), V uses at most r(n) random coins and makes at most q(n) non-adaptive 
queries to locations of m (see Figure 18.1). Then it outputs “1” (for “accept”) or “0” (for 
“reject” ). We use the notation V(x) to denote the random variable representing V’s output 
on input x and with random access to 7. 


Completeness: If x € L then there exists a proof m € {0,1}* such that Pr[V"(x) = 1] = 1. We 
call m the correct proof for x. 


Soundness: If x ¢ L then for every proof m € {0,1}*, Pr[V7 (x) = 1] < 1/2. 


We say that a language L is in PCP(r(n), q(n)) if L has a (c- r(n), d - q(n))-verifier for some 
constants c, d. 

Sometimes we consider verifiers for which the probability “1/2” is replaced by some other number, 
called the soundness parameter. 


THEOREM 18.2 (PCP THEOREM [AS98, ALM*98]) 
NP = PCP(log n, 1). 


Notes: 
1. Without loss of generality, proofs checkable by an (r, q)-verifier contain at most q2” bits. The 
verifier looks at only q places of the proof for any particular choice of its random coins, and 


there are only 2” such choices. Any bit in the proof that is read with 0 probability (i.e., for 
no choice of the random coins) can just be deleted. 
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2. The previous remark implies PCP(r(n), q(n)) E NTIME(20("\™)q(n)). The proofs check- 
able by an (r(n), q(n)-verifier have size at most 20(")g(n). A nondeterministic machine 
could guess the proof in 20())q(n) time, and verify it deterministically by running the ver- 
ifier for all 20(r()) possible choices of its random coin tosses. If the verifier accepts for all 
these possible coin tosses then the nondeterministic machine accepts. 


As a special case, PCP (logn, 1) € NTIME(20(0%87)) = NP: this is the trivial direction of 
the PCP Theorem. 


3. The constant 1/2 in the soundness requirement of Definition 18.1 is arbitrary, in the sense 
that changing it to any other positive constant smaller than 1 will not change the class of 
languages defined. Indeed, a PCP verifier with soundness 1/2 that uses r coins and makes q 
queries can be converted into a PCP verifier using cr coins and cq queries with soundness 
27° by just repeating its execution c times (see Exercise 1). 


EXAMPLE 18.3 
To get a better sense for what a PCP proof system looks like, we sketch two nontrivial PCP 
systems: 


1. The language GNI of pairs of non-isomorphic graphs is in PCP(poly(n),1). Say the input 
for GNI is (Go, G1), where Go, G; have both n nodes. The verifier expects m to contain, for 
each labeled graph H with n nodes, a bit r[H] € {0,1} corresponding to whether H = Go or 
H = Gy (r[A] can be arbitrary if neither case holds). In other words, m is an (exponentially 
long) array of bits indexed by the (adjacency matrix representations of) all possible n-vertex 
graphs. 


The verifier picks b € (0,1) at random and a random permutation. She applies the permuta- 
tion to the vertices of G, to obtain an isomorphic graph, H. She queries the corresponding 
bit of m and accepts iff the bit is b. 


If Go Æ Gj, then clearly a proof m can be constructed which makes the verifier accept with 
probability 1. If Gi = Go, then the probability that amy a makes the verifier accept is at 
most 1/2. 


2. The protocols in Chapter 8 can be used (see Exercise 5) to show that the permanent has 
PCP proof system with polynomial randomness and queries. Once again, the length of the 
proof will be exponential. 


In fact, both of these results are a special case of the following theorem a “scaled-up” version 
of the PCP Theorem which we will not prove. 


THEOREM 18.4 (SCALED-UP PCP, [?, ALM*98, AS98]) 
PCP(poly,1) = NEXP 


18.2. PCP AND HARDNESS OF APPROXIMATION p18.5 (355) 
18.2 PCP and Hardness of Approximation 


The PCP Theorem implies that for many NP optimization problems, computing near-optimal 
solutions is no easier than computing exact solutions. 

We illustrate the notion of approximation algorithms with an example. MAX 3SAT is the prob- 
lem of finding, given a 3CNF Boolean formula vy as input, an assignment that maximizes the number 
of satisfied clauses. This problem is of course NP-hard, because the corresponding decision prob- 
lem, 3SAT, is NP-complete. 


DEFINITION 18.5 
For every 3CNF formula y, define val(p) to be the maximum fraction of clauses that can be satisfied 
by any assignment to y’s variables. In particular, if y is satisfiable then val(y) = 1. 

Let p < 1. An algorithm A is a p-approrimation algorithm for MAX 3SAT if for every 3CNF 
formula y with m clauses, A((p) outputs an assignment satisfying at least p-val(y)m of p's clauses. 


In many practical settings, obtaining an approximate solution to a problem may be almost as 
good as solving it exactly. Moreover, for some computational problems, approximation is much 
easier than an exact solution. 


EXAMPLE 18.6 (1/2-APPROXIMATION FOR MAX 3SAT) 

We describe a polynomial-time algorithm that computes a 1/2-approximation for MAX 3SAT. The 
algorithm assigns values to the variables one by one in a greedy fashion, whereby the ith variable is 
assigned the value that results in satisfying at least 1/2 the clauses in which it appears. Any clause 
that gets satisfied is removed and not considered in assigning values to the remaining variables. 
Clearly, the final assignment will satisfy at least 1/2 of all clauses, which is certainly at least half of 
the maximum that the optimum assignment could satisfy. 

Using semidefinite programming one can also design a polynomial-time (7/8 — €)-approximation 
algorithm for every e > 0 (see references). (Obtaining such a ratio is trivial if we restrict ourselves 
to 83CNF formulae with three distinct variables in each clause. Then a random assignment has 
probability 7/8 to satisfy it, and by linearity of expectation, is expected to satisfy a 7/8 fraction 
of the clauses. This observation can be turned into a simple probabilistic or even deterministic 
7/8-approximation algorithm.) 


For a few problems, one can even design (1 — e)-approximation algorithms for every e > 0. 
Exercise 10 asks you to show this for the NP-complete knapsack problem. 

Researchers are extremely interested in finding the best possible approximation algorithms for 
NP-hard optimization problems. Yet, until the early 1990’s most such questions were wide open. In 
particular, we did not know whether MAX 3SAT has a polynomial-time p-approximation algorithm 
for every p <1. The PCP Theorem has the following Corollary. 


COROLLARY 18.7 
There exists some constant p < 1 such that if there is a polynomial-time p-approximation algorithm 
for MAX 3SAT then P = NP. 
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Later, in Chapter 19, we show a stronger PCP Theorem by Hastad which implies that for 
every € > 0, if there is a polynomial-time (7/8+e)-approximation algorithm for MAX 3SAT then 
P = NP. Hence the approximation algorithm for this problem mentioned in Example 18.6 is very 
likely optimal. The PCP Theorem (and the other PCP theorems that followed it) imply a host 
of such hardness of approximation results for many important problems, often showing that known 
approximation algorithms are optimal. 


18.2.1 Gap-producing reductions 


To prove Corollary 18.7 for some fixed p < 1, it suffices to give a polynomial-time reduction f that 
maps 3CNF formulae to 3CNF' formulae such that: 


p € 3SAT = val(f(p)) = 1 (1) 
p g L= val(f(p)) <p (2) 


After all, if a p-approximation algorithm were to exist for MAX3SAT, then we could use it to 
decide membership of any given formula y in 3SAT by applying reduction f on y and then running 
the approximation algorithm on the resultant 3CNF formula f(y). If val(f(p) = 1, then the 
approximation algorithm would return an assignment that satisfies at least p fraction of the clauses, 
which by property (2) tells us that y € 3SAT. 

Later (in Section 18.2) we show that the PCP Theorem is equivalent to the following Theorem: 


THEOREM 18.8 
There exists some p < 1 and a polynomial-time reduction f satisfying (1) and (2). 


By the discussion above, Theorem 18.8 implies Corollary 18.7 and so rules out a polynomial-time 
p-approximation algorithm for MAX 3SAT (unless P = NP). 


Why doesn’t the Cook-Levin reduction suffice to prove Theorem 18.8? The first thing 
one would try is the reduction from any NP language to 3SAT in the Cook-Levin Theorem (Theo- 
rem 2.10). Unfortunately, it doesn’t give such an f because it does not satisfy property (2): we can 
always satisfy almost all of the clauses in the formulae produced by the reduction (see Exercise 9 
and also the “non-robustness” quote at the start of this chapter). 


18.2.2 Gap problems 


The above discussion motivates the definition of gap problems, a notion implicit in (1) and (2). It 
is also an important concept in the proof of the PCP Theorem itself. 


DEFINITION 18.9 (GAP 3SAT) 
Let p € (0,1). The p-GAP 3SAT problem is to determine, given a 3CNF formula y whether: 


e y is satisfiable, in which case we say y is a YES instance of p-GAP 3SAT. 


e val(y) < p, in which case we say y is a NO instance of p-GAP 3SAT. 
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An algorithm A is said to solve p-GAP 3SAT if A(y) = 1 if y is a YES instance of p-GAP 3SAT 
and A(y) = 0 if y is a NO instance. Note that we do not make any requirement on A(y) if y is 
neither a YES nor a NO instance of p-GAP qCSP. 


Our earlier discussion of the desired reduction f can be formalized as follows. 


DEFINITION 18.10 
Let p € (0,1). We say that p-GAP 3SAT is NP-hard if for every language L there is a polynomial- 
time computable function f such that 


x € L => f(x) isa YES instance of p-GAP 3SAT 
x g L= f(x) is a NO instance of p-GAP 3SAT 


18.2.3 Constraint Satisfaction Problems 


Now we generalize the definition of 3SAT to constraint satisfaction problems (CSP), which allow 
clauses of arbitrary form (instead of just OR of literals) including those depending upon more than 
3 variables. Sometimes the variables are allowed to be non-Boolean. CSPs arise in a variety of 
application domains and play an important role in the proof of the PCP Theorem. 


DEFINITION 18.11 

Let q, W be natural numbers. A gCSPyy instance y is a collection of functions p1,..., Pm (called 
constraints) from {0..W—1}" to {0,1} such that each function p; depends on at most q of its input 
locations. That is, for every i € [m] there exist j1,...,jq € [n] and f : (0..W-1)2 — {0,1} such 
that y;(u) = f(uj,,...,U;,) for every u € {0..W—]}”. 

We say that an assignment u € {0..W—1}” satisfies constraint y; if y;(u) = 1. The fraction of 
constraints satisfied by u is dei pita) and we let val(y) denote the maximum of this value over all 
u € [0..W-1)”. We say that ¢ is satisfiable if val(y) = 1. 

We call q the arity of y and W the alphabet size. If W = 2 we say that y uses a binary alphabet 
and call y a qCSP-instance (dropping the subscript 2). 


EXAMPLE 18.12 
3SAT is the subcase of gCSP yy where q = 3, W = 2, and the constraints are OR’s of the involved 
literals. 

Similarly, the NP-complete problem 3COL can be viewed as a subcase of 2CSP3 instances where 
for each edge (i,j), there is a constraint on the variables u;,u; that is satisfied iff u; 4 uj. The 
graph is 3-colorable iff there is a way to assign a number in {0,1,2} to each variable such that all 
constraints are satisfied. 


Notes: 
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1. We define the size of a qCSPyy-instance y to be the number of constraints m it has. Because 
variables not used by any constraints are redundant, we always assume n < qm. Note that a 
qCSPw instance over n variables with m constraints can be described using O(mn2W?1) bits. 
Usually q, W will be constants (independent of n,m). 


2. As in the case of 3SAT, we can define maximization and gap problems for CSP instances. 
In particular, for any p € (0,1), we define p-GAP gCSPy as the problem of distinguishing 
between a qCSPyy-instance y that is satisfiable (called a YES instance) and an instance p 
with val(p) < p (called a NO instance). As before, we will drop the subscript W in the case 
of a binary alphabet. 


3. The simple greedy approximation algorithm for 3SAT can be generalized for the MAX qCSP 
problem of maximizing the number of satisfied constraints in a given qCSP instance. That 
is, for any qCSPy instance y with m constraints, the algorithm will output an assignment 
satisfying vta in constraints. Thus, unless NP C P, the problem 27 %-GAP qCSP is not NP 
hard. 


18.2.4 An Alternative Formulation of the PCP Theorem 


We now show how the PCP Theorem is equivalent to the NP-hardness of a certain gap version of 
qCSP. Later, we will refer to this equivalence as the “hardness of approximation viewpoint” of the 
PCP Theorem. 


THEOREM 18.13 (PCP THEOREM, ALTERNATIVE FORMULATION) 
There exist constants q € N, p € (0,1) such that p-GAP qCSP is NP-hard. 


We now show Theorem 18.13 is indeed equivalent to the PCP Theorem: 


Theorem 18.2 implies Theorem 18.13. Assume that NP C PCP(logn,1). We will show 
that 1/2-GAP qCSP is NP-hard for some constant q. It is enough to reduce a single NP-complete 
language such as 3SAT to 1/2-GAP qCSP for some constant q. Under our assumption, 3SAT has a 
PCP system in which the verifier V makes a constant number of queries, which we denote by q, 
and uses clogn random coins for some constant c. Given every input x and r € {0,1}°!°8”, define 
Vz r to be the function that on input a proof m outputs 1 if the verifier will accept the proof 7 on 
input x and coins r. Note that Vz, depends on at most q locations. Thus for every x € {0,1}", the 
collection y = {Var tne (0,1}¢!8" is a polynomial-sized gCSP instance. Furthermore, since V runs in 
polynomial-time, the transformation of x to y can also be carried out in polynomial-time. By the 
completeness and soundness of the PCP system, if x € 3SAT then y will satisfy val(p) = 1, while 
if x ¢ 3SAT then y will satisfy val(p) < 1/2. MW 


Theorem 18.13 implies Theorem 18.2. Suppose that p-GAP qCSP is NP-hard for some con- 
stants q,p < 1. Then this easily translates into a PCP system with q queries, p soundness and 
logarithmic randomness for any language L: given an input x, the verifier will run the reduction 
f(x) to obtain a gCSP instance y = {y;}”",. It will expect the proof m to be an assignment to the 
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variables of y, which it will verify by choosing a random i € [m] and checking that y; is satisfied 
(by making q queries). Clearly, if x € L then the verifier will accept with probability 1, while if 
x E Lit will accept with probability at most p. The soundness can be boosted to 1/2 at the expense 
of a constant factor in the randomness and number of queries (see Exercise 1). W 


REMARK 18.14 

Since 3C NF formulas are a special case of 3CSP instances, Theorem 18.8 (p-GAP 3SAT is NP-hard) 
implies Theorem 18.13 (p-GAP gCSP is NP-hard). Below we show Theorem 18.8 is also implied by 
Theorem 18.13, concluding that it is also equivalent to the PCP Theorem. 


It is worth while to review this very useful equivalence between the “proof view” and the 
“hardness of approximation view” of the PCP Theorem: 


PCP verifier (V) — CSP instance (p) 
PCP proof (7) — Assignment to variables (u) 
Length of proof — Number of variables (n) 
Number of queries (q) — Arity of constraints (q) 
Number of random bits (r) <— Logarithm of number of constraints (log m) 
Soundness parameter — Maximum of val(y) for a NO instance 
Theorem 18.2 (NP € PCP(logn,1)) —> Theorem 18.13 (p-GAP gCSP is NP-hard) 


18.2.5 Hardness of Approximation for 3SAT and INDSET. 


The CSP problem allows arbitrary functions to serve as constraints, which may seem somewhat 
artificial. We now show how Theorem 18.13 implies hardness of approximation results for the more 
natural problems of MAX 3SAT (determining the maximum number of clauses satisfiable in a 3SAT 
formula) and MAX INDSET (determining the size of the largest independent set in a given graph). 

The following two lemmas use the PCP Theorem to show that unless P = NP, both MAX 3SAT 
and MAX INDSET are hard to approximate within a factor that is a constantless than 1. ( Sec- 
tion 18.3 proves an even stronger hardness of approximation result for INDSET.) 


LEMMA 18.15 (THEOREM 18.8, RESTATED) 
There exists a constant 0 < p < 1 such that p-GAP 3SAT is NP-hard. 


LEMMA 18.16 
There exist a polynomial-time computable transformation f from 3CNF formulae to graphs such 
that for every 3CNF formula vy, f(y) is an n-vertex graph whose largest independent set has size 


val(y) >. 


PROOF OF LEMMA 18.15: Let e > 0 and q € N be such that by Theorem 18.13, (1—e)-GAP gCSP 
is NP-hard. We show a reduction from (1—e)-GAP gCSP to (1—e')-GAP 3SAT where el > 0 is 
some constant depending on e and q. That is, we will show a polynomial-time function mapping 
YES instances of (1—e)-GAP qCSP to YES instances of (1—e')-GAP 3SAT and NO instances of 
(I—e)-GAP gCSP to NO instances of (1—e’)-GAP 3SAT. 

Let y be a qCSP instance over n variables with m constraints. Each constraint y; of y can be 
expressed as an AND of at most 2% clauses, where each clause is the OR of at most q variables 
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or their negations. Let y’ denote the collection of at most m2% clauses corresponding to all the 
constraints of y. If y is a YES instance of (1—e)-GAP gCSP (i.e., it is satisfiable) then there exists 
an assignment satisfying all the clauses of y”. if p is a NO instance of (1—e)-GAP qCSP then every 
assignment violates at least an e fraction of the constraints of y and hence violates at least an 37 
fraction of the constraints of y. We can use the Cook-Levin technique of Chapter 2 (Theorem 2.10), 
to transform any clause C on q variables on u1,..., Ug to aset C1,..., Cq of clauses over the variables 
U1,..-,Ug and additional auxiliary variables y1, ..., yq such that (1) each clause C; is the OR of at 
most three variables or their negations, (2) if ux,...,ug satisfy C then there is an assignment to 
Y1,- - - , Yg Such that u1,...,Ug, Y1,- - - , Yq Simultaneously satisfy C1,...,Cg and (3) if u1, ... , uy does 
not satisfy C then for every assignment to y1,...,Yq, there is some clause C; that is not satisfies 
by Way ++. Uq, Y) «> -Yq 

Let y” denote the collection of at most qm21 clauses over the n + qm variables obtained in this 
way from vy’. Note that y” is a 3SAT formula. Our reduction will map y to y”. Completeness holds 
since if y was satisfiable then so will be y’ and hence y”. Soundness holds since if every assignment 
violates at least an e fraction of the constraints of p, then every assignment violates at least an 37 
fraction of the constraints of y’, and so every assignment violates at least an pi fraction of the 
constraints of y”. Mi 


PROOF OF LEMMA 18.16: Let y be a 3CNF formula on n variables with m clauses. We define 
a graph G of 7m vertices as follows: we associate a cluster of 7 vertices in G with each clause of 
p. The vertices in cluster associated with a clause C correspond to the 7 possible assignments to 
the three variables C depends on (we call these partial assignments, since they only give values for 
some of the variables). For example, if C is uz V U5 V U7 then the 7 vertices in the cluster associated 
with C correspond to all partial assignments of the form u1 = a, u2 = b, u3 = c for a binary vector 
(a,b,c) 4 (1,1,1). (If C depends on less than three variables we treat one of them as repeated and 
then some of the 7 vertices will correspond to the same assignment.) We put an edge between two 
vertices of G if they correspond to inconsistent partial assignments. Two partial assignments are 
consistent if they give the same value to all the variables they share. For example, the assignment 
uy = 1, u2 = 0, u3 = 0 is inconsistent with the assignment uz = 1, u5 = 0,u7 = 1 because they share 
a variable (ug) to which they give a different value. In addition, we put edges between every two 
vertices that are in the same cluster. 

Clearly transforming y into G can be done in polynomial time. Denote by a(G) to be the 
size of the largest independent set in Œ. We claim that a(G) = val(y)m. For starters, note that 
a(G) > val(y)m. Indeed, let u be the assignment that satisfies val(y~)m clauses. Define a set S as 
follows: for each clause C satisfied by u, put in S the vertex in the cluster associated with C that 
corresponds to the restriction of u to the variables C depends on. Because we only choose vertices 
that correspond to restrictions of the assignment u, no two vertices of S correspond to inconsistent 
assignments and hence S is an independent set of size val(y~)m. 

Suppose that G has an independent set S' of size k. We will use S to construct an assignment 
u satisfying k clauses of p, thus showing that val(y)m > a(G). We define u as follows: for every 
i € [n], if there is a vertex in S whose partial assignment gives a value a to u;, then set uj = a; 
otherwise set u; = 0. This is well defined because S is an independent set, and each variable u; 
can get at most a single value by assignments corresponding to vertices in S. On the other hand, 
because we put all the edges within each cluster, S can contain at most a single vertex in each 
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cluster, and hence there are k distinct cluster with members in S. By our definition of u it satisfies 
all the clauses associated with these clusters. Mi 


REMARK 18.17 

In Chapter 2, we defined L’ to be NP-hard if every L € NP reduces to L’. The reduction 
was a polynomial-time function f such that x € L = f(x) € L’. In all cases, we proved that 
x € L > f(x) € L’ by showing a way to map a certificate to the fact that x € L to a certificate 
to the fact that x’ € L’. Although the definition of a Karp reduction does not require that this 
mapping is efficient, it often turned out that the proof did provide a way to compute this mapping 
in polynomial time. The way we proved that f(a) € L' > x € L was by showing a way to map a 
certificate to the fact that x’ € L’ to a certificate to the fact that x € L. Once again, the proofs 
typically yield an efficient way to compute this mapping. 

A similar thing happens in the gap preserving reductions used in the proofs of Lemmas 18.15 
and 18.16 and elsewhere in this chapter. When reducing from, say, p-GAP qCSP to p’-GAP 3SAT 
we show a function f that maps a CSP instance (p to a 3SAT instance Y satisfying the following 
two properties: 


Completeness We can map a satisfying assignment of ọ to a satisfying assignment to y 


Soundness Given any assignment that satisfies more than a p' fraction of 7’s clauses, we can map 
it back into an assignment satisfying more than a p fraction of y’s constraints. 


18.3 n-°-approximation of independent set is NP-hard. 


We now show a much stronger hardness of approximation result for the independent set (INDSET) 
problem than Lemma 18.16. Namely, we show that there exists a constant ô € (0,1) such that 
unless P = NP, there is no polynomial-time n°-approximation algorithm for INDSET. That is, we 
show that if there is a polynomial-time algorithm A that given an n-vertex graph G outputs an 
independent set of size at least E (where opt is the size of the largest independent set in G) then 
P = NP. We note that an even stronger result is known: the constant 6 can be made arbitrarily 
close to 1 [?, ?]. This factor is almost optimal since the independent set problem has a trivial 
n-approximation algorithm: output a single vertex. 

Our main tool will be the notion of expander graphs (see Note 18.18 and Chapter ??). Expander 
graphs will also be used in the proof of PCP Theorem itself. We use here the following property 
of expanders: 


LEMMA 18.19 

Let G = (V, E) be a A-expander graph for some A € (0,1). Let S be a subset of V with |S| = B|V| 
for some 3 € (0,1). Let (X,,..., Xp) be a tuple of random variables denoting the vertices of a 
uniformly chosen ((-1)-step path in G. Then, 


(8 —2d)* < Pr Mic Xi € S] < (8+ 2d)* 


The upper bound of Lemma 18.19 is implied by Theorem ??; we omit the proof of the lower 
bound. 
The hardness result for independent set follows by combining the following lemma with Lemma 18.16: 
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NOTE 18.18 (EXPANDER GRAPHS) 
Expander graphs are described in Chapter ??. We define there a parameter 
A(G) € [0,1], for every regular graph G (see Definition 7.25). The main 
property we need in this chapter is that for every regular graph G = (V, E) 
and every S C V with |S| < |V|/2, 


IS| /1 AMG) 
P € E S| < 
ua SESS |V| \2 t (a 


Another property we use is that \(G“) = AG)” for every £ € N, where G* is 
obtained by taking the adjacency matrix of G to the #” power (i.e., an edge 
in G“ corresponds to an (€-1)-step path in G). 

For every c € (0,1), we call a regular graph G satisfying AG) < ca c- 
expander graph. If c < 0.9, we drop the prefix c and simply call G an 
expander graph. (The choice of the constant 0.9 is arbitrary.) As shown 
in Chapter ??, for every constant c € (0,1) there is a constant d and an 
algorithm that given input n € N, runs in poly(n) time and outputs the 
adjacency matrix of an n-vertex d-regular c-expander (see Theorem 16.32). 


LEMMA 18.20 
For every À > 0 there is a polynomial-time computable reduction f that maps every n-vertex graph 
G into an m-vertex graph H such that 


(&(G) — 24) 87 < &(H) < (a(G) + 2d)'8" 
where 4(G) is equal to the fractional size of the largest independent set in G. 


Recall that Lemma 18.16 shows that there are some constants 5,€ € (0,1) such that it is NP- 
hard to tell whether a given graph G satisfies (1) 4(G) > 8 or (2) a(G) < (1—e)6. By applying 
to G the reduction of Lemma 18.20 with parameter A = (Ge/8 we get that in case (1), a(H) > 
(8—Be/4)0en = (B(1—e/4))"E7, and in case (2), &(H) < ((1—e)8+Pe/4)87 = (8(1 —0.75€)) 8”. 
We get that the gap between the two cases is equal to c!°8” for some c > 1 which is equal to m? 
for some ô > 0 (where m = poly(n) is the number of vertices in H). 


PROOF OF LEMMA 18.20: Let G, A be as in the lemma’s statement. We let K be an n-vertex 
A-expander of degree d (we can obtain such a graph in polynomial-time, see Note 18.18). We will 
map G into a graph H of ndl87-1 vertices in the following way: 


e The vertices of H correspond to all the (log n—1)-step paths in the A-expander K. 


e We put an edge between two vertices u, v of H corresponding to the paths (u1,..., Wogn) and 
(U1,-.-,Vlogn) if there exists an edge in G between two vertices in the set {u1,..., Wogn, U1, +++ 5 Vlogn}- 


18.4. NP C PCP(POLY(N),1): PCP BASED UPON WALSH-HADAMARD CODEp18.13 (363) 


A subset T of H’s vertices corresponds to a subset of log n-tuples of numbers in [n], which we 
can identify as tuples of vertices in G. We let V(T') denote the set of all the vertices appearing in 
one of the tuples of T. Note that in this notation, T is an independent set in H if and only if V(T) 
is an independent set of G. Thus for every independent set T in H, we have that |V(T)| < a(G)n 
and hence by the upper bound of Lemma 18.19, T takes up less than an (&( H) + 2A)!°8” fraction 
of H’s vertices. On the other hand, if we let S be the independent set of G of size 4(G)n then by 
the lower bound of Lemma. 18.19, an (@ — 2A)'°8” fraction of H’s vertices correspond to paths fully 
contained in S, implying that 4(H) > (4(G) — 24)87. W 


18.4 NP C PCP(poly(n),1): PCP based upon Walsh-Hadamard 
code 


We now prove a weaker version of the PCP theorem, showing that every NP statement has an 
exponentially-long proof that can be locally tested by only looking at a constant number of bits. In 
addition to giving a taste of how one proves PCP Theorems, this section builds up to a stronger 
Corollary 18.26, which will be used in the proof of the PCP theorem. 


THEOREM 18.21 
NP C PCP(poly(n), 1) 


We prove this theorem by designing an appropriate verifier for an NP-complete language. The 
verifier expects the proof to contain an encoded version of the usual certificate. The verifier checks 
such an encoded certificate by simple probabilistic tests. 


18.4.1 Tool: Linearity Testing and the Walsh-Hadamard Code 


We use the Walsh-Hadamard code (see Section 17.5, though the treatment here is self-contained). 
It is a way to encode bit strings of length n by linear functions in n variables over GF(2); namely, 
the function WH : (0,1)* — (0,1)* mapping a string u € (0,1)” to the truth table of the function 
x ++ u Ox, where for x,y € ([0,1)” we define xO y = > ;_, ziyi (mod 2). Note that this is a 
very inefficient encoding method: an n-bit string u € (0, 1)” is encoded using |WH(u)| = 2” bits. 
If f € (0,1) is equal to WH(u) for some u then we say that f is a Walsh-Hadamard codeword. 
Such a string f € (0,1) can also be viewed as a function from (0,1)” to {0,1}. 

The Walsh-Hadamard code is an error correcting code with minimum distance 1/2, by which we 
mean that for every u % u’ € {0,1}", the encodings WH(u) and WH(u) differ in half the bits. This 
follows from the familiar random subsum principle (Claim A.5) since exactly half of the strings 
x € {0,1}” satisfy u © x 4 u' Ox. Now we talk about local tests for the Walsh-Hadamard code 
(i.e., tests making only O(1) queries). 


Local testing of Walsh-Hadamard code. Suppose we are given access to a function f : 
{0,1}" — {0,1} and want to test whether or not f is actually a codeword of Walsh-Hadamard. 
Since the Walsh-Hadamard codewords are precisely the set of all linear functions from (0, 1)” to 
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{0,1}, we can test f by checking that 


f(x+y) = f(x) + fly) (4) 


for all the 2?” pairs x,y € {0,1}" (where “+” on the left side of (pcp:eq:lintest) denotes vector 
addition over GF(2)” and on the right side denotes addition over GF(2)). 

But can we test f by querying it in only a constant number of places? Clearly, if f is not linear 
but very close to being a linear function (e.g., if f is obtained by modifying a linear function on 
a very small fraction of its inputs) then such a local test will not be able to distinguish f from a 
linear function. Thus we set our goal on a test that on one hand accepts every linear function, and 
on the other hand rejects with high probability every function that is far from linear. It turns out 
that the natural test of choosing x, y at random and verifying (4) achieves this goal: 


DEFINITION 18.22 

Let p € [0,1]. We say that f,g: [0,1)” — {0,1} are p-close if Prxep{o,1} [S (x) = g(x)] > p. We 
say that f is p-close to a linear function if there exists a linear function g such that f and g are 
p-close. 


THEOREM 18.23 (LINEARITY TESTING [?]) 
Let f : 10,1)” — {0,1} be such that 


Fay) = fa) + fy) =p 


Pr 
x,y€r{0,1}” 
for some p > 1/2. Then f is p-close to a linear function. 


We defer the proof of Theorem 18.23 to Section 19.3 of the next chapter. For every ô € (0, 1/2), 
we can obtain a linearity test that rejects with probability at least 1/2 every function that is not 
(1-0)-close to a linear function, by testing Condition (4) repeatedly O(1/9) times with independent 
randomness. We call such a test a (1—ô) -linearity test. 


Local decoding of Walsh-Hadamard code. Suppose that for ô < ł the function f : {0,1}" = 
{0,1} is (1-6)-close to some linear function f. Because every two linear functions differ on half of 
their inputs, the function f is uniquely determined by f. Suppose we are given x € {0,1}" and 
random access to f. Can we obtain the value f (x) using only a constant number of queries? The 
naive answer is that since most x’s satisfy f(x) = f(x), we should be able to learn f(x) with good 
probability by making only the single query x to f. The problem is that x could very well be one of 
the places where f and f differ. Fortunately, there is still a simple way to learn f (x) while making 
only two queries to f: 


1. Choose x’ Ep {0,1}”. 
2. Set x” =x+x’. 
3. Let y’ = f(x’) and y” = f(x”). 


4. Output y! + y”. 
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Since both x’ and x” are individually uniformly distributed (even though they are dependent), 
by the union bound with probability at least 1 — 28 we have y” = f(x’) and y” = f(x”). Yet by 
the linearity of f, f(x) = f(x’ +x”) = f(x’) + f(x”), and hence with at least 1 — 26 probability 
f(x) = y'+y”.* This technique is called local decoding of the Walsh-Hadamard code since it allows 
to recover any bit of the correct codeword (the linear function f) from a corrupted version (the 
function f) while making only a constant number of queries. It is also known as self correction of 
the Walsh-Hadamard code. 


18.4.2 Proof of Theorem 18.21 


We will show a (poly(n), 1)-verifier proof system for a particular NP-complete language L. The 
result that NP C PCP(poly(n), 1) follows since every NP language is reducible to L. The NP- 
complete language L we use is QUADEQ, the language of systems of quadratic equations over 
GF(2) = {0,1} that are satisfiable. 


EXAMPLE 18.24 
The following is an instance of QUADEQ over the variables u;z,...,us: 


uuz + u3zu4 + urus = 1 
u2U3 + UYU, = 0 


u]U4 + ugus + ugua = 1 


This instance is satisfiable since the all-1 assignment satisfies all the equations. 


More generally, an instance of QUADEQ over the variables u;,...,un is of the form AU = b, 
where U is the n?-dimensional vector whose (i, j)” entry is ujuj, A is an m x n? matrix and 
b € {0,1}. In other words, U is the tensor product u & u, where x Y y for a pair of vectors 
x,y € {0,1}” denotes the n?-dimensional vector (or n x n matrix) whose (i, j) entry is xiyj. For 
every i, j € [n] with i < j, the entry Az, {i,j} is the coefficient of uju,; in the kt? equation (we identify 
[n?] with [n] x [n] in some canonical way). The vector b consists of the right hand side of the m 
equations. Since u; = (u;)? in GF(2), we can assume the equations do not contain terms of the 
form uz. 

Thus a satisfying assignment consists of u1,u2,...,Un E GF(2) such that its tensor product 
U=u®u satisfies AU = b. We leave it as Exercise 12 to show that QUADEQ, the language of all 
satisfiable instances, is indeed NP-complete. 

We now describe the PCP system for QUADEQ. Let 4,b be an instance of QUADEQ and 
suppose that A,b is satisfiable by an assignment u € {0,1}”. The correct PCP proof m for A,b 
will consist of the Walsh-Hadamard encoding for u and the Walsh-Hadamard encoding for u Y u, 
by which we mean that we will design the PCP verifier in a way ensuring that it accepts proofs 


3We use here the fact that over GF(2), a +b = a — b. 
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WH(u) $ WH(u®u) z 


Figure 18.2: The PCP proof that a set of quadratic equations is satisfiable consists of WH(u) and WH(u 8 u) for 
some vector u. The verifier first checks that the proof is close to having this form, and then uses the local decoder of 
the Walsh-Hadamard code to ensure that u is a solution for the quadratic equation instance. 


of this form with probability one, satisfying the completeness condition. (Note that m is of length 
249%) 

Below, we repeatedly use the following fact: 
RANDOM SUBSUM PRINCIPLE: If u v then for at least 1/2 the choices of x, uOx A vOx. Realize 
that x can be viewed as a random subset of indices in [1,...,n] and the principle says that with 
probability 1/2 the sum of the u;'s over this index set is different from the corresponding sum of vi’s. 


2 
2742" 


The verifier. The verifier V gets access to a proof m € {0,1} , which we interpret as a pair 


of functions f : {0,1}" — {0,1} and g : {0, i => {0,1}. 
Step 1: Check that f, g are linear functions. 


As already noted, this isn’t something that the verifier can check per se using local tests. Instead, 
the verifier performs a 0.99-linearity test on both f,g, and rejects the proof at once if either test 
fails. 

Thus, if either of f, g is not 0.99-close to a linear function, then V rejects with high probability. 
Therefore for the rest of the procedure we can assume that there exist two linear functions f : 
{0,1}" — {0,1} and 7: wir — {0,1} such that f is 0.99-close to f, and g is 0.99-close to g. 
(Note: in a correct proof, the tests succeed with probability 1 and f=fandg= 9.) 

In fact, we will assume that for Steps 2 and 3, the verifier can query f,g at any desired point. 
The reason is that local decoding allows the verifier to recover any desired value of f,g with good 
probability, and Steps 2 and 3 will only use a small (less than 15) number of queries to f,g. Thus 
with high probability (say > 0.9) local decoding will succeed on all these queries. 

NOTATION: To simplify notation in the rest of the procedure we use f,g for f,g respectively. 
Furthermore, we assume both f and g are linear, and thus they must encode some strings u € (0, 1)" 
and w € (0, 1y”. In other words, f,g are the functions given by f(r) =u©r and g(z) =w0z. 
Step 2: Verify that g encodes u 9 u, where u € (0,1)” is the string encoded by f. 

Verifier V does the following test 3 times: “Choose r,r’ independently at random from (0,1)”, 


and if f(r) f(r’) 4 g(r 8 r”) then halt and reject.” 
In a correct proof, w = u® u, so 


f(r) f(r’) = y uri `> ujt), = 


¿e[n] jeln] 


y ujujrir; =(u®@u)O(r er’), 
i,jeln] 
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which in the correct proof is equal to g(r & r’). Thus Step 2 never rejects a correct proof. 

Suppose now that, unlike the case of the correct proof, w 4 u@u. We claim that in each of the 
three trials V will halt and reject with probability at least L. (Thus the probability of rejecting in 
at least one trial is at least 1 — (3/4)? = 37/64.) Indeed, let W be an n x n matrix with the same 
entries as w, let U be the n x n matrix such that U;,; = ujuj and think of r as a row vector and r’ 
as a column vector. In this notation, 


gr@r)=wo(rer)= > wigrity =rWr' 
¿,jeln] 


fS = uor) (uor) = o> uir) O ujt) = yA ujujrirj = UF 
i=1 j=1 i,j¡e[n] 

And V rejects if rWr’ 4 rUr'. The random subsum principle implies that if W Æ U then at 
least 1/2 of all r satisfy rW # rU. Applying the random subsum principle for each such r, we 
conclude that at least 1/2 the r’ satisfy rWr’ 4 rUr’. We conclude that the test rejects for at least 
1/4 of all pairs r, r’. 


Step 3: Verify that f encodes a satisfying assignment. 
Using all that has been verified about f,g in the previous two steps, it is easy to check that any 
particular equation, say the kth equation of the input, is satisfied by u, namely, 


y Ak (ij) Willy = by. (5) 
ij 

Denoting by z the n? dimensional vector (Ax,(,7)) (where i, j vary over [1..n]), we see that the 
left hand side is nothing but g(z). Since the verifier knows A; (ij) and bj, it simply queries g at z 
and checks that g(z) = bp. 

The drawback of the above idea is that in order to check that u satisfies the entire system, 
the verifier needs to make a query to g for each k = 1,2,...,m, whereas the number of queries is 
required to be independent of m. Luckily, we can use the random subsum principle again! The 
verifier takes a random subset of the equations and computes their sum mod 2. (In other words, 
for k = 1,2,...,m multiply the equation in (5) by a random bit and take the sum.) This sum is a 
new quadratic equation, and the random subsum principle implies that if u does not satisfy even 
one equation in the original system, then with probability at least 1/2 it will not satisfy this new 
equation. The verifier checks that u satisfies this new equation. 

(Actually, the above test has to be repeated twice to ensure that if u does not satisfy the system, 
then Step 3 rejects with probability at least 3/4.) 


18.4.3 PCP’s of proximity 


Theorem 18.21 says that (exponential-sized) certificates for NP languages can be checked by ex- 
amining only O(1) bits in them. The proof actually yields a somewhat stronger result, which will 
be used in the proof of the PCP Theorem. This concerns the following scenario: we hold a circuit 
C in our hands that has n input wires. Somebody holds a satisfying assignment u. He writes down 
WH(u) as well as another string 7 for us. We do a probabilistic test on this by examining O(1) bits 
in these strings, and at the end we are convinced of this fact. 
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Concatenation test. First we need to point out a property of Walsh-Hadamard codes and a 
related concatenation test. In this setting, we are given two linear functions f, g that encode strings 
of lengths n and n + m respectively. We have to check by examining only O(1) bits in f, g that if 
u and v are the strings encoded by f, g (that is, f = WH(u) and h = WH(v)) then u is the same 
as the first n bits of v. By the random subsum principle, the following simple test rejects with 
probability 1/2 if this is not the case. Pick a random x € (0,1)”, and denote by X € GF(2)7+" 
the string whose first n bits are x and the remaining bits are all-0. Verify that f(X) = g(x). 
With this test in hand, we can prove the following corollary. 


COROLLARY 18.25 (EXPONENTIAL-SIZED PCP OF PROXIMITY.) 
There exists a verifier V that given any circuit C of size m and with n inputs has the following 
property: 


1. Ifu € {0,1}" is a satisfying assignment for circuit C, then there is a string ra of size 2P oly(m) 
such that V accepts WH(u) o 72 with probability 1. (Here o denotes concatenation.) 


2. For every strings 7,72 € [0,1)*, where mı has 2” bits, if V accepts mı o 12 with probability 
at least 1/2, then mı is 0.99-close to WH(u) for some u that satisfies C. 


3. V uses poly(m) random bits and examines only O(1) bits in the provided strings. 


PROOF: One looks at the proof of NP-completeness of QUADEQ to realize that given circuit C 
with n input wires and size m, it yields an instance of QUADEQ of size O(m) such that u € (0, 1)" 
satisfies the circuit iff there is a string v of size M = O(m) such that uo v satisfies the instance of 
QUADEQ. (Note that we are thinking of u both as a string of bits that is an input to C and as a 
string over GF(2)” that is a partial assignment to the variables in the instance of QUADEQ.) 

The verifier expects 72 to contain whatever our verifier of Theorem 18.21 expects in the proof for 
this instance of QUADEQ, namely, a linear function f that is WH(w), and another linear function 
g that is WH(w ® w) where w satisfies the QUADEQ instance. The verifier checks these functions 
as described in the proof of Theorem 18.21. 

However, in the current setting our verifer is also given a string 7 € (0, 3”. Think of this as 
a function h:GF(2)” — GF(2). The verifier checks that h is 0.99-close to a linear function, say h. 
Then to check that f encodes a string whose first n bits are the same as the string encoded by h, 
the verifier does a concatenation test. 

Clearly, the verifier only reads O(1) bits overall. W 


The following Corollary is also similarly proven and is the one that will actually be used later. 
It concerns a similar situation as above, except the inputs to the circuit C are thought of as the 
concatenation of two strings of lengths n,,n2 respectively where n = nj + no. 


COROLLARY 18.26 (PCP OF PROXIMITY WHEN ASSIGNMENT IS IN TWO PIECES) 
There exists a verifier V that given any circuit C with n input wires and size m and also two 
numbers n1, no such that ny + ng =n has the following property: 


1. If uy € {0,1}™ ,ug € {0,1} is such that uy o ua is a satisfying assignment for circuit C, 
then there is a string 73 of size 2P°l¥™) such that V accepts WH(u¡) o WH(ug) o m3 with 
probability 1. 
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2. For every strings 11,72, 713 € {0,1}", where mı and ra have 2" and 2” bits respectively, if 
V accepts Tı o T2 o 73 with probability at least 1/2, then 7,72 are 0.99-close to WH(u1), 
WH(uz2) respectively for some u1, u2 such that uy o uz is a satisfying assignment for circuit 


C. 


3. V uses poly(m) random bits and examines only O(1) bits in the provided strings. 


18.5 Proof of the PCP Theorem. 


As we have seen, the PCP Theorem is equivalent to Theorem 18.13, stating that p-GAP gCSP is 
NP-hard for some constants q and p < 1. Consider the case that p = 1—e where e is not necessarily 
a constant but can be a function of m (the number of constraints). Since the number of satisfied 
constraints is always a whole number, if y is unsatisfiable then val(y) < 1 — 1/m. Hence, the gap 
problem (1—1/m)-GAP 3CSP is a generalization of 3SAT and is NP hard. The idea behind the 
proof is to start with this observation, and iteratively show that (1—e)-GAP gCSP is NP-hard for 
larger and larger values of e, until e is as large as some absolute constant independent of m. This 
is formalized in the following lemma. 


DEFINITION 18.27 

Let f be a function mapping CSP instances to CSP instances. We say that f is a CL-reduction 
(short for complete linear-blowup reduction) if it is polynomial-time computable and for every CSP 
instance y with m constraints, satisfies: 


Completeness: If ọ is satisfiable then so is f(y). 


Linear blowup: The new qCSP instance f(y) has at most Cm constraints and alphabet W, 
where C and W can depend on the arity and the alphabet size of y (but not on the number 
of constraints or variables). 


LEMMA 18.28 (PCP MAIN LEMMA) 
There exist constants qo > 3, €o > 0, and a CL-reduction f such that for every qyCSP-instance p 
with binary alphabet, and every e < €o, the instance Y = f(y) is a qoCSP (over binary alphabet) 
satisfying 

val(p) < 1—e => val(W) < 1 — 2e 


Lemma 18.28 can be succinctly described as follows: 


Arity | Alphabet | Constraints | Value 
Original qo binary m l-e 

y y y y 
Lemma 18.28 | qo binary Cm 1-—2Le 


This Lemma allows us to easily prove the PCP Theorem. 
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Proving Theorem 18.2 from Lemma 18.28. Let go > 3 be as stated in Lemma 18.28. As 
already observed, the decision problem ggCSP is NP-hard. To prove the PCP Theorem we give 
a reduction from this problem to GAP qgCSP. Let p be a qCSP instance. Let m be the number 
of constraints in y. If y is satisfiable then val(y) = 1 and otherwise val(p) < 1—1/m. We use 
Lemma 18.28 to amplify this gap. Specifically, apply the function f obtained by Lemma 18.28 to 
p a total of logm times. We get an instance y such that if y is satisfiable then so is 4, but if p 
is not satisfiable (and so val(y) < 1— 1/m) then val(w) < 1 — minf2eo, 1 — 28M/m) = 1 — 260. 
Note that the size of w is at most Cm, which is polynomial in m. Thus we have obtained 
a gap-preserving reduction from L to the (1-2e9)-GAP qgCSP problem, and the PCP theorem is 
proved. W 


The rest of this section proves Lemma 18.28 by combining two transformations: the first trans- 
formation amplifies the gap (i.e., fraction of violated constraints) of a given CSP instance, at the 
expense of increasing the alphabet size. The second transformation reduces back the alphabet to 
binary, at the expense of a modest reduction in the gap. The transformations are described in the 
next two lemmas. 


LEMMA 18.29 (GAP AMPLIFICATION [?]) 
For every £ € N, there exists a CL-reduction gg such that for every CSP instance y with binary 
alphabet, the instance w = g¢(v) has has arity only 2 (but over a non-binary alphabet) and satisfies: 


val(y) < 1 — e = val(w) < 1 — le 


for every € < €o where ey > 0 is a number depending only on £ and the arity q of the original 
instance (p. 


LEMMA 18.30 (ALPHABET REDUCTION) 

There exists a constant qy and a CL- reduction h such that for every CSP instance y, if y had 
arity two over a (possibly non-binary) alphabet {0..W—1} then a = h(p) has arity qo over a binary 
alphabet and satisfies: 


val(y) < 1—e => val(h(p)) < 1 — €/3 


Lemmas 18.29 and 18.30 together imply Lemma 18.28 by setting f(y) = h(gs(p)). Indeed, if 
y was satisfiable then so will f(y). If val(p) < 1 — e, for e < co (where ey the value obtained in 
Lemma 18.29 for £ = 6, q = qo) then val(g6(y)) < 1 — 6e and hence val(h(g6(y))) < 1— 2e. This 
composition is described in the following table: 


Arity | Alphabet | Constraints | Value 
Original do binary m l-e 

y y y y 
Lemma 18.29 | 2 W Cm 1 — 6e 

y y y y 
Lemma 18.30 | qo binary C'Cm 1 — 2e 
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18.5.1 Gap Amplification: Proof of Lemma 18.29 


To prove Lemma 18.29, we need to exhibit a function g that maps a gCSP instance to a 2CSP yy 
instance over a larger alphabet {0..W—1} in a way that increases the fraction of violated constraints. 
We will show that we may assume without loss of generality that the instance of gCSP has a 
specific form. To describe this we need a definition. 
We will assume that the instance satisfies the following properties, since we can give a simple 
CL-reduction from gCSP to this special type of gCSP. (See the “Technical Notes” section at the 
end of the chapter.) We will call such instances “nice.” 


Property 1: The arity q is 2 (though the alphabet may be nonbinary). 


Property 2: Let the constraint graph of w be the graph G with vertex set [n] where for every 
constraint of p depending on the variables u;, uj, the graph G has the edge (i, j). We allow G 
to have parallel edges and self-loops. Then G is d-regular for some constant d (independent 
of the alphabet size) and at every node, half the edges incident to it are self-loops. 


Property 3: The constraint graph is an expander. 


The rest of the proof consists of a “powering” operation for nice 2CSP instances. This is 
described in the following Lemma. 


LEMMA 18.31 (POWERING) 
Let y be a 2CSPyy instance satisfying Properties 1 through 3. For every number t, there is an 
instance of 2CSP w such that: 


1. y is a 2CSPy-instance with alphabet size W’ < we", where d denote the degree of 1p's 
constraint graph. The instance 4* has ditvin constraints, where n is the number of variables 


in Y. 
2. If 1) is satisfiable then so is y*. 


3. For every e < Pure if val(w) < 1 — e then val(1/%) <1-e for e = nda €. 


4. The formula y* is computable from % in time polynomial in m and we, 


PROOF: Let y be a 2CSPyy-instance with n variables and m = nd constraints, and as before let G 
denote the constraint graph of w. 

The formula 4* will have the same number of variables as 7. The new variables y = Y1,...,Yn 
take values over an alphabet of size W” = W*” and thus a value of a new variable yi is a d-tuple 
of values in {0..W—1}. We will think of this tuple as giving a value in {0..W—1} to every old variable 
uj Where j can be reached from u; using a path of at most t + Vt steps in G (see Figure 18.3). 
In other words, the tuple contains an assignment for every uj such that j is in the ball of radius 
t+ yt and center i in G. For this reason, we will often think of an assignment to y; as “claiming” a 
certain value for uj. (Of course, another variable y, could claim a different value for uj.) Note that 
since G has degree d, the size of each such ball is no more than d'+Vt+1 and hence this information 
can indeed be encoded using an alphabet of size W’. 
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Figure 18.3: An assignment to the formula y consists of n variables over an alphabet of size less than we”, where 
each variable encodes the restriction of an assignment of y to the variables that are in some ball of radius t + vt in 
w’s constraint graph. Note that an assignment y1,...,Yn to Y” may be inconsistent in the sense that if i falls in the 
intersection of two such balls centered at k and k’, then Yk may claim a different value for u; than the value claimed 


by yx. 


For every (2+ 1)-step path p = (i1,...,%2¢42) in G, we have one corresponding constraint Cp in 
yt (see Figure 18.4). The constraint Cp depends on the variables y; and yj,,, and outputs FALSE 
if (and only if) there is some j € [2t + 1] such that: 


1. i; is in the t + vVt-radius ball around i1. 
2. ij+1 is in the t+ /t-radius ball around 19449 


3. If w denotes the value y; claims for uj, and w’ denotes the value yi», claims for u;,,, , then 
the pair (w, w’) violates the constraint in y that depends on u;, and %;,, y. 


Figure 18.4: 7° has one constraint for every path of length 2t + 1 in 7’s constraint graph, checking that the views 
of the balls centered on the path’s two endpoints are consistent with one another and the constraints of 1. 


A few observations are in order. First, the time to produce such an assignment is polynomial 
in m and We, so part 4 of Lemma 18.29 is trivial. 
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Second, for every assignment to u1, U2,..., Un we can “lift” it to a canonical assignment to 
Y1,- --,Yn by simply assigning to each y; the vector of values assumed by u,;’s that lie in a ball of 
radius t + vt and center i in G. If the assignment to the uj’S was a satisfying assignment for Y, 
then this canonical assignment satisfies 7", since it will satisfy all constraints encountered in walks 
of length 2t + 1 in G. Thus part 2 of Lemma 18.29 is also trivial. 

This leaves part 3 of the Lemma, the most difficult part. We have to show that if val(y) < 1—e 


then every assignment to the y;'s satisfies at most 1 — ¢' fraction of constraints in 7’, where e < in 


and el = mr. This is tricky since an assignment to the y;’s does not correspond to any obvious 
assignment for the u,'s: for each uj, different values could be claimed for it by different y;’s. The 
intuition will be to show that these inconsistencies among the y;'s can't happen too often (at least 
if the assignment to the y;’s satisfies 1 — e” constraints in 4”). 

From now on, let us fix some arbitrary assignment y = Y1,...,Yn to 4v%s variables. The following 
notion is key. 

The plurality assignment: For every variable u; of y, we define the random variable Z; over 
{0,...,W — 1} to be the result of the following process: starting from the vertex i, take a t step 
random walk in G to reach a vertex k, and output the value that yz claims for u;. We let z; denote 
the plurality (i.e., most likely) value of Z;. If more than one value is most likely, we break ties 
arbitrarily. This assignment is called a plurality assignment (see Figure 18.5). Note that Z; = 2; 
with probability at least 1/W. 


Figure 18.5: An assignment y for 4* induces a plurality assignment u for w in the following way: u; gets the most 
likely value that is claimed for it by yx, where k is obtained by taking a t-step random walk from 7 in the constraint 
graph of y. 


Since val(w) < 1 — e, every assignment for 4 fails to satisfy 1 — e fraction of the constraints, 
and this is therefore also true for the plurality assignment. Hence there exists a set F of em = end 
constraints in y that are violated by the assignment z = z1,..., Zn. We will use this set F to show 
that at least an e fraction of 1)%s constraints are violated by the assignment y. 

Why did we define the plurality assignment z in this way? The reason is illustrated by the 
following claim, showing that for every edge f = (i, i’) of G, among all paths that contain the edge 
f somewhere in their “midsection”, most paths are such that the endpoints of the path claim the 
plurality values for u; and uj. 
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CLAIM 18.32 
For every edge f = (i,1') in G define the event Bj, over the set of (2t+1)-step paths in G to contain 
all paths (i1,..., 12142) satisfying: 


e f is the j} edge in the path. That is, f = (ij, ij+1). 

e y;, Claims the plurality value for u;. 

e Yin Claims the plurality value for uy. 

Let 6 = mir . Then for every j € ae no tb dvi}, Pr[B; s] > -r 


PROOF: First, note that because G is regular, the jt? edge of a random path is a random edge, and 
hence the probability that f is the jt? edge on the path is equal to h. Thus, we need to prove 
that, 


1 
Pr[endpoints claim plurality values for u;, uy (resp.)|f is j edge] > 22 (6) 


We start with the case j = t + 1. In this case (6) holds essentially by definition: the left-hand 
side of (6) is equal to the probability that the event that the endpoints claim the plurality for these 
variables happens for a path obtained by joining a random t-step path from į to a random t-step 
path from 7’. Let k be the endpoint of the first path and k’ be the endpoint of the second path. Let 
W; be the distribution of the value that yẹ claims for u;, where k is chosen as above, and similarly 
define W; to be the distribution of the value that yx claims for uj. Note that since k and k’ are 
chosen independently, the random variables W; and Wy are independent. Yet by definition the 
distribution of W; identical to the distribution Z;, while the distribution of Wy is identical to Zy. 
Thus, 


Pr[endpoints claim plurality values for u;, uy (resp.)|f is j edge] = 


In the case that 7 4 2t+1 we need to consider the probability of the event that endpoints claim 
the plurality values happening for a path obtained by joining a random t — 1 + j-step path from 
i to a random t+ 1 — j-step path from 7’ (see Figure 18.6). Again we denote by k the endpoint 
of the first path, and by k’ the endpoint of the second path, by W; the value yx claims for ui 
and by W; the value yz. claims for uy. As before, W; and Wy are independent. However, this 
time W; and Z; may not be identically distributed. Fortunately, we can show that they are almost 
identically distributed, in other words, the distributions are statistically close. Specifically, because 
half of the constraints involving each variable are self loops, we can think of a t-step random walk 
from a vertex i as follows: (1) throw t coins and let S; denote the number of the coins that came 
up “heads” (2) take S “real” (non self-loop) steps on the graph. Note that the endpoint of a 
t-step random walk and a t'-step random walk will be identically distributed if in Step (1) the 
variables S; and Sy turn out to be the same number. Thus, the statistical distance of the endpoint 
of a t-step random walk and a t/-step random walk is bounded by the statistical distance of S 
and Sy where Sp denotes the binomial distribution of the sum of £ balanced independent coins. 
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However, the distributions S; and S, 57 are within statistical distance at most 100 for every 0, t 
(see Exercise 15) and hence in our case W; and Wy are qg -close to Z; and Zy respectively. Thus 
| Pre [Wi = zi] — Pr[Z; = zil| < gop, | Prel[We = ev] — Pr[Zy = 2w]] < ip which proves (6) also 
for the case j 4 2t+1. W 


Figure 18.6: By definition, if we take two t-step random walks from two neighbors i and i’, then the respective 
endpoints will claim the plurality assignments for u; and uz with probability more than 1/(2W?). Because half 
the edges of every vertex in G have self loops, this happens even if the walks are not of length t but of length in 
[t — evt, t + vt] for sufficiently small e. 


Recall that F is the set of constraints of y (=edges in G) violated by the plurality assignment 
z. Therefore, if f € F and j € {t, weeny tt ôvt} then all the paths in B; f correspond to constraints 
of yt that are violated by the assignment y. Therefore, we might hope that the fraction of violated 
constraints in y* is at least the sum of Pr[B; f] for every f € F and j € (+, Lo. tb dyt). Tf this 


were the case we'd be done since Claim 18.32 implies that this sum is at least dviend = byte >g 


However, this is inaaccurate since we are overcounting paths that contain more than one such 
violation (i.e., paths which are in the intersection of B; ¿ and By fr for (j, f) 4 (5', f'))- To bound 
the effect of this overcounting we prove the following claim: 


CLAIM 18.33 
For every k € N and set F of edges with |F| = end for € < zy, 


Y  PriB;¿M By p] < 30kde (7) 
JJ At.t+k} 


, 


EF 
PEDF) 


PROOF: Only one edge can be the j*” edge of a path, and so for every f Æ f', Pr[B; f M B; pr] = 0. 
Thus the left-hand side of (7) simplifies to 


Y.) Pena (8) 


GAH la IAS 
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Let A; be the event that the j’” edge is in the set F. We get that (8) is equal to 


Pr[A; N Aj] =D > Pr[A; N Aj] (9) 
JAI C{t..t +k} j<j €{t..t+k} 

Let S be the set of at most den vertices that are adjacent to an edge in F. For j’ < j, Pr[AjNAj’] 
is bounded by the probability that a random (j’—j)-step path in G has both endpoints in S, or in 
other words that a random edge in the graph G!’~J has both endpoints in S. Using the fact that 
MGÍ=5) = \(G)"-5 < 0.971, this probability is bounded by de(de + 0.911-71) (see Note 18.18). 
Plugging this into (9) and using the formula for summation of arithmetic series, we get that: 


2 Y Pr[Ajn Ay] < 
i<j C{t,...,t+k} 
t+k-j 


2 Y. SO de(de + 0.9") < 


j€{t,...,t+k}i=1 


2k? de + 2kde X 0.9 < 2k?d?e? + 20kde < 30kde 
i=l 


where the last inequality follows from e < Li a 


Wrapping up. Claims 18.32 and 18.33 together imply that 


XO PriB;,] > 6Vte5pz (10) 
je{t..t+5vt} 
JEF 
XO Pr[By,p A By p] < 308vtde (11) 
jij €{ t..tovt} 
Ff EF 
LATAS) 


But (10) and (11) together imply that if p is a random constraint of 4* then 


Vte 
Pr[p violated by y] > Prl U B;,ş] = 240dW2 
je{t.t+ovt} 
fer 


where the last inequality is implied by the following technical claim: 


CLAIM 18.34 


Let Ay,..., An ben subsets of some set U satisfying > ¿| A¡NAj| < C X; 145] for some number 
CEN. Then, 


“a 5 Xi l4:l 
= 4C 
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PROOF: We make 2C copies of every element u € U to obtain a set U with |U| = 2C|U|. Now for 
every subset A; C U, we obtain A; C U as follows: for every u € A;, we choose at random one of 
the 2C copies to put in A;. Note that |4;| = |A;|. For every i,j € [n], u € Ayn Aj, we denote by 
Li ju the indicator random variable that is equal to 1 if we made the same choice for the copy of u 
in A; and Aj, and equal to 0 otherwise. Since E[J; ju] = 305 


A A Ai N A; 
Ella] = Y) Ellija]= anal 
uEA¡NA; 


and 
14044] = Zian 
i i<j Di ~ Oo 


This means that there exists some choice of Ai, eee , A; such that 


n n 


Y lä = $ 144] > 2) 14,0 Ay 


i=1 i=1 i<j 


which by the inclusion-exclusion principle (see Section ??) means that |U; A;| > 25; |Ail. But 
because there is a natural 2C-to-one mapping from U;A; to U;A; we get that 
| Via Ail J Vir Ail _ Dosey 144] 


mn, A;| > = 
[Wi Al 2 “Gq 2 46 4C 


Since el < Ide this proves the lemma. Ml 
240dW? > P 


18.5.2 Alphabet Reduction: Proof of Lemma 18.30 


Lemma 18.30 is actually a simple consequence of Corollary 18.26, once we restate it using our 
“gCSP view” of PCP systems. 


COROLLARY 18.35 (qCSP VIEW OF PCP OF PROXIMITY.) 
There exists positive integer qy and an exponential-time transformation that given any circuit C of 
size m and and n inputs and two numbers n1,n2 such that ni + ng = n produces an instance Yo 
of qCSP of size 2POLYM) over a binary alphabet such that: 


1. The variables can be thought of as being partitioned into three sets 7, 72,73 where 7, has 
2™ variables and mə has 2” variables. 


2. If u, € {0,1}™ , ug € [0,1P? is such that uy o uz is a satisfying assignment for circuit C, 
then there is a string 73 of size 2Poly(m) such that WH(u1) o WH(uz) o m3 satisfies wo. 
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3. For every strings 71,72, 713 € {0,1}", where mı and ra have 2”! and 2” bits respectively, if 
Tı O T2 O m3 Satisfy at least 1/2 the constraints of wo, then 11,72 are 0.99-close to WH(uj), 
WH(uz2) respectively for some u1, u2 such that uy o uz is a satisfying assignment for circuit 


C. 


Now we are ready to prove Lemma 18.30. 


PROOF OF LEMMA 18.30: Suppose the given arity 2 formula y has n variables u¡,uz,..., Un, 
alphabet {0..W—1l} and N constraints C1, C2,...,Cy. Think of each variable as taking values that 
are bit strings in {0,1}*, where k = [logW]. Then if constraint Cy involves variables say Ui, Uj 
we may think of it as a circuit applied to the bit strings representing ui, uj where the constraint 
is said to be satisfied iff this circuit outputs 1. Say m is an upperbound on the size of this circuit 
over all constraints. Note that m is at most 22* < W4. We will assume without loss of generality 
that all circuits have the same size. 

If we apply the transformation of Corollary 18.35 to this circuit we obtain an instance of qoCSP, 
say Yo, The strings u;,u; get replaced by strings of variables U;, U; of size 22 < 2” that take 
values over a binary alphabet. We also get a new set of variables that play the role analogous to 
73 in the statement of Corollary 18.35. We call these new variables II. 

Our reduction consists of applying the above transformation to each constraint, and taking 
the union of the ggCSP instances thus obtained. However, it is important that these new qgCSP 
instances share variables, in the following way: for each old variable u;, there is a string of new 
variables U; of size 22 and for each constraint C, that contains u;i, the new qgCSP instance Yo, 
uses this string U;. (Note though that the I; variables are used only in Yc, and never reused.) 
This completes the description of the new qgCSP instance y (see Figure 18.7). Let us see that it 
works. 


Original instance: 


C; C, c 
constraints: A pa RN, 
variables: uy Uy ][Ug ] eee Un 
(over alphabet [W]) 
LL 
Pee 


Transformed instance: 


clu: 


constraints: ¿e 


variables: | ||[[[ [LIL LL e | fas 
(over alphabet {0.1}) U, =WH(u, ) Us=WH (uy) U,=WH(u,) 4 Im 


Figure 18.7: The alphabet reduction transformation maps a 2CSP instance y over alphabet {0..W—1} into a 
qCSP instance ~ over the binary alphabet. Each variable of y is mapped to a block of binary variables that in the 
correct assignment will contain the Walsh-Hadamard encoding of this variable. Each constraint Cy of p depending 
on variables u;,u; is mapped to a cluster of constraints corresponding to all the PCP of proximity constraints for 
Ce. These constraint depend on the encoding of u; and uj, and on additional auxiliary variables that in the correct 
assignment contain the PCP of proximity proof that these are indeed encoding of values that make the constraint 
Ce true. 
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Suppose the original instance py was satisfiable by an assignment uz,...,Un. Then we can 
produce a satisfying assignment for Y by using part 2 of Corollary 18.35, so that for each constraint 
Cı involving u;, uz, the encodings WH(u;), WH(uj) act as 7,72 and then we extend these via a 
suitable string 73 into a satisfying assignment for Yc. 

On the other hand if val(p) < 1 — e then we show that val(yw) < 1 — e/2. Consider any 
assignment U1, Uz,... Un, I1l;,..., In to the variables of y. We “decode” it to an assignment 
for y as follows. For each i = 1,2,...,n, if the assignment to U; is 0.99-close to a linear function, 
let u; be the string encoded by this linear function, and otherwise let u; be some arbitrary string. 
Since val(y) < 1 — e, this new assignment fails to satisfy at least e fraction of constraints in y. For 
each constraint C, of y that is not satisfied by this assignment, we show that at least 1/2 of the 
constraints in Wc, are not satisfied by the original assignment, which leads to the conclusion that 
val(1) < 1 — €/2. Indeed, suppose C; involves u;, uj. Then u; o uz is not a satisfying assignment to 
circuit Cj, so part 3 of Corollary 18.35 implies that regardless of the value of variables in IN, the 
assignment U; o uj o II; must have failed to satisfy at least 1/2 the constraints of Yg,. Mi 


18.6 The original proof of the PCP Theorem. 


The original proof of the PCP Theorem, which resisted simplification for over a decade, used 
algebraic encodings and ideas that are complicated versions of our proof of Theorem 18.21. (Indeed, 
Theorem 18.21 is the only part of the original proof that still survives in our writeup.) Instead of the 
linear functions used in Welsh-Hadamard code, they use low degree multivariate polynomials. These 
allow procedures analogous to the linearity test and local decoding, though the proofs of correctness 
are a fair bit harder. The alphabet reduction is also somewhat more complicated. The crucial part 
of Dinur’s simpler proof, the one given here, is the gap amplification lemma (Lemma 18.29) that 
allows to iteratively improve the soundness parameter of the PCP from very close to 1 to being 
bounded away from 1 by some positive constant. This general strategy is somewhat reminiscent 
of the zig-zag construction of expander graphs and Reingold’s deterministic logspace algorithm for 
undirect connectivity described in Chapter ??. 


Chapter notes 


Problems 


$1 Prove that for every two functions r,q : N > N and constant s < 1, changing the constant in 
the soundness condition in Definition 18.1 from 1/2 to s will not change the class PCP(r, q). 


§2 Prove that for every two functions r, q : N — N and constant c > 1/2, changing the constant in 
the completeness condition in Definition 18.1 from 1 to c will not change the class PCP(r, q). 


$3 Prove that any language L that has a PCP-verifier using r coins and q adaptive queries also 
has a standard (i.e., non-adaptive) verifier using r coins and 2% queries. 


§4 Prove that PCP(0,logn) = P. Prove that PCP(0, poly(n)) = NP. 
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86 


87 


88 


89 


Let L be the language of matrices A over GF(2) satisfying perm(A) = 1 (see Chapters ?? 
and 8). Prove that L is in PCP(poly(n), poly(n)). 


Show that if SAT € PCP(r(n), 1) for r(n) = o(log n) then P = NP. (Thus the PCP Theorem 
is probably optimal up to constant factors.) 


(A simple PCP Theorem using logspace verifiers) Using the fact that a correct tableau can 
be verified in logspace, we saw the following exact characterization of NP: 


NP = {L: there is a logspace machine M s.t x € L iff dy : M accepts (x, y).}. 


Note that M has two-way access to y. 


Let L-PCP(r(n)) be the class of languages whose membership proofs can be probabilistically 
checked by a logspace machine that uses O(r(n)) random bits but makes only one pass over 
the proof. (To use the terminology from above, it has 2-way access to x but l-way access 
to y.) As in the PCP setting, “probabilistic checking of membership proofs” means that for 
x E L there is a proof y that the machine accepts with probability 1 and if not, the machine 
rejects with probability at least 1/2. Show that NP = L-PCP(logn). Don’t assume the PCP 
Theorem! 
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(This simple PCP Theorem is implicit in Lipton [Lip90]. The suggested proof is due to van 
Melkebeek. ) 


Suppose we define J — PCP(r(n)) similarly to L — PCP(r(n)), except the verifier is only 
allowed to read O(r(n)) successive bits in the membership proof. (It can decide which bits 
to read.) Then show that J — PC P(logn) C L. 


Prove that there is an NP-language L and x ¢ L such that f(x) is a 3SAT formula with 
m constraints having an assignment satisfying more than m — m?’ of them, where f is the 
reduction from f to 3SAT obtained by the proof of the Cook-Levin theorem (Section 2.3). 
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§10 Show a poly(n, 1/e)-time 1 + e approximation algorithm for the knapsack problem. That is, 
show an algorithm that given n +1 numbers aj,...,@, € N (each represented by at most n 


bits) and k € [n], finds a set S C [n] with |S| < k such that D¡¿g ai > $ where 


+e 


opt= max Say 
SC[n],|S|<k * 
ics 
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$11 Show a polynomial-time algorithm that given a satisfiable 2CSP-instance y (over binary 
alphabet) finds a satisfying assignment for y. 


$12 Prove that QUADEQ is NP-complete. 
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$13 Prove that if Z, U are two n x n matrices over GF(2) such that Z 4 U then 


[rZr 4rUr” > 


ml rR 


Pr 
rjr/ERr{0,1}” 
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$14 Show a deterministic poly(n, 2%)-time algorithm that given a qCSP-instance y (over binary 
alphabet) with m clauses outputs an assignment satisfying m/2? of these assignment. 
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$15 Let 5; be the binomial distribution over t balanced coins. That is, Pr[S; = k] = 2% Prove 
that for every ô < 1, the statistical distance of S; and S,,5 7 is at most 10e. 
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$16 The long-code for a set {0,...,W — 1) is the function LC: {0,...,W — 1} — (0, pes such 
that for every i € {0..W—l} and a function f : {0..W—1} — {0,1}, (where we identify f with 
an index in [2”]) the ft? position of LC(i), denoted by LC(i) p, is f (i). We say that a function 


L: {0, Na — {0,1} is a long-code codeword if L = LC(i) for some i € {0..W—l}. 
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(a) Prove that LC is an error-correcting code with distance half. That is, for every i 4 j € 


[0..W—1), the fractional Hamming distance of LC(i) and LC(¿) is half. 


(b) Prove that LC is locally-decodable. That is, show an algorithm that given random access 


Ne 


to a function L : 2490" > {0,1} that is (1-e)-close to LC(i) and f : {0..W—l} — {0,1} 
outputs LC(i) with probability at least 0.9 while making at most 2 queries to L. 


Let L = LC(i) for some i € {0..W—1}. Prove the for every f : {0..W-—l} — {0,1}, 


L(f) = 1-L(f), where f is the negation of f (i.e. , f(i) = 1— f (i) for every i € {0..W4}). 


Let T be an algorithm that given random access to a function L : 209 _, {0,1}, does 
the following: 
i. Choose f to be a random function from {0..W—l} — (0,1). 
ii. If L(f) =1 then output TRUE. 
iii. Otherwise, choose g : (0..W—1) — {0,1} as follows: for every i € {0..W—l}, if 
f(z) = 0 then set g(¿) = 0 and otherwise set g(i) to be a random value in {0,1}. 
iv. If L(g) = 0 then output TRUE; otherwise output FALSE. 
Prove that if L is a long-code codeword (i.e., L = LC(i) for some i) then T outputs 
TRUE with probability one. 
Prove that if L is a linear function that is non-zero and not a longcode codeword then 


T outputs TRUE with probability at most 0.9. 


Prove that LC is locally testable. That is, show an algorithm that given random access 
to a function L : {0,1} — {0,1} outputs TRUE with probability one if L is a long- 
code codeword and outputs FALSE with probability at least 1/2 if L is not 0.9-close to a 
long-code codeword, while making at most a constant number of queries to L. 
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(f) Using the test above, give an alternative proof for the Alphabet Reduction Lemma 


(Lemma 18.30). 
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Omitted proofs 


The preprocessing step transforms a qCSP-instance y into a “nice” 2CSP-instance y through the 
following three claims: 


CLAIM 18.36 
There is a CL- reduction mapping any qCSP instance y into a 2CSPoq instance y such that 


val(y) < 1 — e = val(w) < 1 — e/q 


PROOF: Given a qCSP-instance y over n variables u1,..., Un with m constraints, we construct the 
following 2CSPə, formula y over the variables u1,...,Un,Y1;---, Ym. Intuitively, the y; variables 
will hold the restriction of the assignment to the q variables used by the ¿* constraint, and we will 
add constraints to check consistency: that is to make sure that if the it” constraint depends on the 
variable uj then uj is indeed given a value consistent with y;. Specifically, for every y; of p that 
depends on the variables u1,..., uq, we add q constraints {Wij} je[q) Where Y; ¿(y;, uj) is true iff y; 
encodes an assignment to u1,..., uy satisfying y; and uj is in {0,1} and agrees with the assignment 
Yi. Note that the number of constraints in y is qm. 

Clearly, if y is satisfiable then so is y. Suppose that val(p) < 1—e and let w1,..., Uk, Y1,---;Ym 
be any assignment to the variables of y. There exists a set S C [m] of size at least em such that 
the constraint y; is violated by the assignment u1,..., ug. For any i € S there must be at least one 
j € [q] such that the constraint 4; j is violated. Mi 


CLAIM 18.37 
There is an absolute constant d and a CL- reduction mapping any 2CSP yy instance y into a 2CSPyw 
instance Y such that 

val(p) < 1—e => val(w) < 1 — e/(100Wa). 


and the constraint graph of w is d-regular. That is, every variable in y appears in exactly d 
constraints. 


PROOF: Let y be a 2CSPyy instance, and let {Gn en be an explicit family of d-regular expanders. 
Our goal is to ensure that each variable appears in y at most d +1 times (if a variable appears 
less than that, we can always add artificial constraints that touch only this variable). Suppose 
that u; is a variable that appears in k constraints for some n > 1. We will change u; into k 
variables y},...,y*, and use a different variable of the form y] in the place of u; in each constraint 


u; originally appeared in. We will also add a constraint requiring that y is equal to y! "for every 
edge (j, 7’) in the graph Gk. We do this process for every variable in the original instance, until 
each variable appears in at most d equality constraints and one original constraint. We call the 
resulting 2CSP-instance Y. Note that if y has m constraints then y will have at most m + dm 
constraints. 

Clearly, if y is satisfiable then so is 4. Suppose that val(y) < 1—e and let y be any assignment 
to the variables of 7. We need to show that y violates at least ¡557 of the constraints of y. Recall 
that for each variable u; that appears k times in y, the assignment y has k variables yl,..., yl. 
We compute an assignment u to y’s variables as follows: u; is assigned the plurality value of 
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Y, bes „yE . We define t; to be the number of y! ’s that disagree with this plurality value. Note that 
0 < t; < k(1—1/W) (where W is the alphabet size). If )7;_,t; > §m then we are done. Indeed, 
by (3) (see Note 18.18), in this case we will have at least Dj, my > im equality constraints 
that are violated. 

Suppose now that X; ti < Gm. Since val(p) < 1—e, there is a set S of at least em constraints 
violated in y by the plurality assignment u. All of these constraints are also present in 4 and since 
we assume )>;_/ ti < Fm, at most half of them are given a different value by the assignment y than 
the value given by u. Thus the assignment y violates at least 5m constraints in y. Mi 


CLAIM 18.38 
There is an absolute constant d and a CL-reduction mapping any 2CSPy instance y with d'-regular 
constraint graph for d > d' into a 2CSPy instance y such that 


val(y) < 1—e= val(w) < 1 — e/(10d) 


and the constraint graph of y is a 4d-regular expander, with half the edges coming out of each 
vertex being self-loops. 


PROOF: There is a constant d and an explicit family {Gn nen of graphs such that for every n, Gn 
is a d-regular n-vertex 0.1-expander graph (See Note 18.18). 

Let y be a 2CSP-instance as in the claim’s statement. By adding self loops, we can assume that 
the constraint graph has degree d (this can at worst decrease the gap by factor of d). We now add 
“null” constraints (constraints that always accept) for every edge in the graph G,,. In addition, we 
add 2d null constraints forming self-loops for each vertex. We denote by y the resulting instance. 
Adding these null constraints reduces the fraction of violated constraints by a factor at most four. 
Moreover, because any regular graph H satisfies A(H) < 1 and because of A's subadditivity (see 
Exercise 11, Chapter ??), A(Y) < 3+ 4A(Gn) < 0.9 where by A(w) we denote the parameter A of 
w’s constraint graph. Mi 


Chapter 19 


More PCP Theorems and the Fourier 
Transform Technique 


The PCP Theorem has several direct applications in complexity theory, in particular showing 
that unless P = NP, many NP optimization problems can not be approximated in polynomial- 
time to within arbitrary precision. However, for some applications, the standard PCP Theorem 
does not suffice, and we need stronger (or simply different) “PCP Theorems”. In this chapter we 
survey some of these results and their proofs. The Fourier transform technique turned out to be 
especially useful in advanced PCP constructions, and in other areas in theoretical computer science. 
We describe the technique and show two of its applications. First, we use Fourier transforms to 
prove the correctness of the linearity testing algorithm of Section 18.4, completing the proof of the 
PCP Theorem. We then use it to prove a stronger PCP Theorem due to Hastad, showing tight 
inapproximability results for many important problems, including MAX 3SAT. 


19.1 Parallel Repetition of PCP’s 


Recall that the soundness parameter of a PCP system is the probability that the verifier may 
accept a false statement. Definition 18.1 specified the soundness parameter to be 1/2, but as we 
noted, it can be reduced to an arbitrary small constant by increasing the number of queries. Yet 
for some applications we need a system with, say, three queries, but an arbitrarily small constant 
soundness parameter. Raz has shown that this can be achieved if we consider systems with non 
binary alphabet. (For a finite set S, we say that a PCP verifier uses alphabet S if it takes as input 
a proof string m in S*.) The idea is simple and natural: use parallel repetition. That is, we take a 
PCP verifier V and run £ independent copies of it, to obtain a new verifier V° such that a query 
of V“ is the concatenation of the £ queries of V, and an answer is a concatenation of the £ answers. 
(So, if the original verifier V used proofs over, say, the binary alphabet, then the verifier V* will 
use the alphabet {0, 1}°.) The verifier V“ accepts the proof only of all the £ executions of V accept. 
Formally, we define parallel repetition as follows: 


DEFINITION 19.1 (PARALLEL REPETITION) 
Let S be a finite set. Let V be a PCP verifier using alphabet S and let £ € N. The £-times parallel 
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Original V | Parallel repeated V* | Sequential repeated V°% 
Alphabet size W we W 
Proof size m mé m 
Random coins used r lr lr 
Number of queries q q lq 
Completeness probability 1 1 1 
soundness parameter 1-6 =o" (1-6)? 


Table 19.1: Parameters of ¢-times parallel repeated verifier V° vs. parameters for sequential repetition. 


repeated V is the verifier V‘ that operates as follows: 


1. V£ uses the alphabet S = S°. We denote the input proof string to V? by #. 


2. Let q denote the number of queries V makes. On any input x, V‘ chooses l independent 
random tapes r!,...,r' for V, and runs V on the input and these tapes to obtain £ sets of q 


queries 
-1 -1 -1 
li, 12, , tq 
2 -2 2 
a, 2, tq 
£ -£ £ 
li; 1, 3 tg 
3. V* makes q queries i4,... , iq to TT where i; is (ij, pia , if) (under a suitable encoding of N' into 
N). 
4. For j € [q], denote (as, ve , 05) = 7(i;). The verifier V‘ accepts if and only for every k € [4, 
the verifier V on random tape rz, accepts when given the responses ak seed ae 


REMARK 19.2 

For every input zx, if there is a proof m such that on input x, the verifier V accepts m with probability 
one, then there is a proof 7 such that on input x, the verifier Vf accepts 7 with probability one. 
Namely, for every ¢-tuple of positions i',...,7°, the proof 7 contains the tuple (r[i*],...,r[1*). 
Note that |#| = |r]*. 


Why is it called “parallel repetition”? We call the verifier V‘ the parallel repeated version 
of V to contrast with sequential repetition. If V is a PCP verifier and £ € N, we say that ¢-times 
sequentially repeated V, denoted V**%, is the verifier that chooses £ random tapes for V, then 
makes the ql queries corresponding to these tapes one after the other, and accepts only if all the 
instances accept. Note that V**% uses the same alphabet as V, and uses proofs of the same size. 
The relation between the parameters of V, Vf and V**% is described in Table 19.1. 
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It is a simple exercise to show that if V’s soundness parameter was 1 — ô then V°% soundness 
parameter will be equal to (1-9). One may expect the soundness parameter of the parallel repeated 
verifier V^ to also be (1—6)*. It turns out this is not the case (there is a known counterexample [?]), 
however the soundness parameter does decay exponentially with the number of repetitions: 


THEOREM 19.3 (PARALLEL REPETITION LEMMA, [RAZ98]) 
There exist constants a,b (independent of £ but depending on the alphabet size used and number 
of queries) such that the soundness parameter of V° is at most (1 — 9%) 


We omit the proof of Theorem 19.3 for lack of space. Roughly speaking, the reason analyzing 
soundness of V“ is so hard is the following: for every tuple (i1,...,i¢), the corresponding position 
in the proof for V“ is “supposed” to consist of the values mfi] o --+7[ig] where m is some proof for 
V. However, a priori, we do not know if the proof satisfies this property. It may be that the proof 
is inconsistent and that two tuples containing the it” position “claim” a different assignment for 
Ti]. 


REMARK 19.4 

The Gap Amplification Lemma (Lemma 18.29) of the previous chapter has a similar flavor, in the 
sense that it also reduced the soundness parameter at the expense of an increase in the alphabet 
size. However, that lemma assumed that the soundness parameter is very close to 1, and its proof 
does not seem to generalize for soundness parameters smaller than 1/2. We note that a weaker 
version of Theorem 19.3, with a somewhat simpler proof, was obtained by Feige and Kilian [?]. 
This weaker version is sufficient for many applications, including for Hastad’s 3-query PCP theorem 
(see Section 19.2 below). 


19.2 Hastad’s 3-bit PCP Theorem 


In most cases, the PCP Theorem does not immediately answer the question of exactly how well can 
we approximate a given optimization problem (even assuming P # NP). For example, the PCP 
Theorem implies that if P 4 NP then MAX3SAT cannot be c-approximated in polynomial-time 
for some constant p < 1. But if one follows closely the proof of Theorem 18.13, this constant p 
turns out to be very close to one, and in particular it is larger than 0.999. On the other hand, 
as we saw in Example 18.6, there is a known 7/8-approximation algorithm for MAX 3SAT. What 
is the true “approximation complexity” of this problem? In particular, is there a polynomial-time 
0.9-approximation algorithm for it? Similar questions are the motivation behind many stronger 
PCP theorems. In particular, the following theorem by Hastad implies that for every e > 0 there 
is no polynomial-time (7/8+e)-approximation for MAX 3SAT unless P = NP: 


THEOREM 19.5 (HASTAD’S 3-BIT PCP [?]) 

For every e > 0 and every language L € NP there is a PCP-verifier V for L making three (binary) 

queries having completeness probability at least 1 — € and soundness parameter at most 1/2 + e. 
Moreover, the test used by V are linear. That is, given a proof n € {0,1}"", V chooses a triple 

(i1, 12,73) € [m]? and b € {0,1} according to some distribution and accepts iff Ti + Ti, + Tig = b 

(mod 2). 
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Theorem 19.5 immediately implies that the problem MAX E3LIN is NP-hard to 1/2+e-approximate 
for every € > 0, where MAX B3 LIN is the problem of finding a solution maximizing the number of 
satisfied equations among a given system of linear equations over GF(2), with each equation con- 
taining at most 3 variables. Note that this hardness of approximation result is tight since a random 
assignment is expected to satisfy half of the equations. Also note that finding out whether there 
exists a solution satisfying all of the equations can be done in polynomial-time using Gaussian 
elimination (and hence the imperfect completeness in Theorem 19.5 is inherent). 

The result for MAX 3SAT is obtained by the following corollary: 


COROLLARY 19.6 
For every e > 0, computing (7/8+e)-approximation to MAX 3SAT is NP-hard. 


PROOF: We reduce MAX E3LIN to MAX3SAT. Take any instance of MAX E3LIN where we are 
interested in determining whether (1—e) fraction of the equations can be satisfied or at most 1/2+ € 
are. Represent each linear constraint by four 3CNF" clauses in the obvious way. For example, the 
linear constraint a+b+c =0 (mod 2) is equivalent to the clauses (@VbVc), (aVbV c), (aVbVz), (av 
bV@). If a,b,c satisfy the linear constraint, they satisfy all 4 clauses and otherwise they satisfy at 
most 3 clauses. We conclude that in one case at least (1 — e) fraction of clauses are simultaneously 
satisfiable, and in the other case at most 1 — (5 — €) x j = z — y fraction are. The ratio between 
the two cases tends to 7/8 as e decreases. Since Theorem 19.5 implies that distinguishing between 


the two cases is NP-hard for every constant e, the result follows. Mi 


19.3 Tool: the Fourier transform technique 


The continuous Fourier transform is extremely useful in mathematics and engineering. Likewise, 
the discrete Fourier transform has found many uses in algorithms and complexity, in particular for 
constructing and analyzing PCP’s. The Fourier transform technique for PCP’s involves calculating 
the maximum acceptance probability of the verifier using Fourier analysis of the functions presented 
in the proof string. It is delicate enough to give “tight” inapproximability results for MAX INDSET, 
MAX 3SAT, and many other problems. 

To introduce the technique we start with a simple example: analysis of the linearity test over 
GF(2) (i.e., proof of Theorem 18.23). We then introduce the Long Code and show how to test for 
membership in it. These ideas are then used to prove Hastad’s 3-bit PCP Theorem. 


19.3.1 Fourier transform over GF(2)” 


The Fourier transform over GF(2)” is a tool to study functions on the Boolean hypercube. In this 
chapter, it will be useful to use the set (+1,—1) = {+1} instead of {0,1}. To transform {0,1} 
to {+1}, we use the mapping b +> (—1)? (i.e., 0> +1 , 1+ —1). Thus we write the hypercube 
as {+1}" instead of the more usual {0,1}". Note this maps the XOR. operation (i.e., addition in 
GF(2)) into the multiplication operation. 


The set of functions from [+1)” to R defines a 2”-dimensional Hilbert space (see Section ??) 
as follows. Addition and multiplication by a scalar are defined in the natural way: (f + g)(x) = 
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f(x) + 9(x) and (af)(x) = af(x) for every f,g: {+1}" — R, a € R. We define the inner product 
of two functions f, g, denoted (f,g), to be Exes+ij"[ f(x) 9(x)]. 

The standard basis for this space is the set {e€x}xe{+1}", Where ex(y) is equal to 1 if y = x, 
and equal to 0 otherwise. This is an orthonormal basis, and every function f : {+1}" — R can be 
represented in this basis as f = Y”, axex. For every x € {+1}”, the coefficient ax is equal to f(x). 
The Fourier basis for this space is the set {Xa}acjn} Where xa(x) = Ilica ti (Xg is the constant 
1 function). These correspond to the linear functions over GF(2). To see this, note that every 
linear function of the form b +> a@b (with a,b € {0,1}") is mapped by our transformation to the 
function taking x € {+1}” to Il; s4. aj=1 Ti: 

The Fourier basis is indeed an orthonormal basis for the Hilbert space. Indeed, the random 
subsum principle implies that for every a, 3 C [n], (xa, X8) = ĝa, g where ôa g is equal to 1 iff a = 8 
and equal to 0 otherwise. This means that every function f : (+1)” — R can be represented as 
f= rar faXa- We call Fo the at” Fourier coefficient of f. 

We will often use the following simple lemma: 


LEMMA 19.7 
Every two functions f,g:{+1}" — R satisfy 


1. (f, 9) = Xa Fado 


2. (Parseval’s Identity) (f, f) =o, f2 


PROOF: The second property follows from the first. To prove the first we expand 
a B 
y fado Uta xe) = y faúcÓa.s = S fade 


a,b a,b 
E 


EXAMPLE 19.8 
Some examples for the Fourier transform of particular functions: 


1. If f(u1,u2,..., Un) = ui (i.e., f is a coordinate function, a concept we will see again soon) 
then f = xi} and so fy} = 1 and fa = 0 for a F {i}. 


2. If f isa random boolean function on n bits, then each fa is a random variable that is a sum of 
2” binomial variables (equally likely to be 1,—1) and hence looks like a normally distributed 
variable with standard deviation 2”/? and mean 0. Thus with high probability, all 2” Fourier 

poly(n) polvoj; 


2 > 9n/2 


coefficients have values in [ 
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The connection to PCPs: High level view 


In the PCP context we are interested in Boolean-valued functions, i.e., those from GF'(2)" to GF (2). 
Under our transformation these are mapped to functions from {+1}" to {+1}. Thus, we say that 
: f {+1}”" — R is Boolean if f(x) € {+1} for every x € {+1}”. Note that if f is Boolean then 
(F, f) = Ex[f(x)?] = 1. 

On a high level, we use the Fourier transform in the soundness proofs for PCP’s to show that 
if the verifier accepts a proof m with high probability then m is “close to” being “well-formed” 
(where the precise meaning of “close-to” and “well-formed” is context dependent). Technically, 
we will often be able to relate the acceptance probability of the verifier to an expectation of the 
form (f,g) = Ex[f(x)g(x)], where f and g are Boolean functions arising from the proof. We then 
use techniques similar to those used to prove Lemma 19.7 to relate this acceptance probability 
to the Fourier coefficients of f,g, allowing us to argue that if the verifier’s test accepts with high 
probability, then f and g have few relatively large Fourier coefficients. This will provide us with 
some nontrivial useful information about f and g, since in a “generic” or random function, all the 
Fourier coefficient are small and roughly equal. 


19.3.2 Analysis of the linearity test over GF(2) 


We will now prove Theorem 18.23, thus completing the proof of the PCP Theorem. Recall that 
the linearity test is provided a function f : GF(2)” — GF(2) and has to determine whether f 
has significant agreement with a linear function. To do this it picks x,y € GF(2)” randomly and 
accepts iff f(x +y) = f(x) + f(y). 

Now we rephrase this test using {+1} instead of GF(2), so linear functions turn into Fourier basis 
functions. For every two vectors x, y € (+1)”, we denote by xy their componentwise multiplication. 
That is, xy = (11Y1,...,TnYn). Note that for every basis function ¥q(xy) = Ya(X)Xa(y). 


For two Boolean functions f,g, (f,g) is equal to the fraction of inputs on which they agree 
minus the fraction of inputs on which they disagree. It follows that for every e € [0,1] and functions 
fig: {+1}" — {+1}, f has agreement 5 + 5 with g iff (f,g) = e. Thus, if f has a large Fourier 
coefficient then it has significant agreement with some Fourier basis function, or in the GF(2) 
worldview, f is close to some linear function. This means that Theorem 18.23 can be rephrased as 
follows: 


THEOREM 19.9 
Suppose that f : {+1}" — {+1} satisfies Przy[f(xy) = f(x)f(y)] > Ty e. Then, there is some 
a.C [n] such fa de 


PROOF: We can rephrase the hypothesis as Ex y[f(xy)f(x)f(y)] > (4 +€) -— (4 — €) = 2e. We note 
that from now on we do not need f to be Boolean, but merely to satisfy (f, f) = 1. 
Expressing f by its Fourier expansion, 


2e < Exylf(xy) F&F] = EN O taxa YO aae dx (9). 
~ B p 
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Since Xa(XY) = Xa(X)Xa(y) this becomes 


= Exy[ Y fafs fixa) xay) xex) (y)! 
a By 


Using linearity of expectation: 
fafafExy xa) xa (Y)XG (9 x (y) 


=X 
Si, Ê Ex xa(x)x0(X)] Ey xaly) xy) 
apy 


(because x, y are independent). 


By orthonormality Ex[xa(x)x8(x)] = 40,8, so we simplify to 
= 2, Ta 


< (ng fa) 074) 


Since > a f2 = (f, f) = 1, this expression is at most maxa { fabs Hence maxo A > 2e and the 
theorem is proved. W 


19.3.3 Coordinate functions, Long code and its testing 


Let W € N. We say that f : {+1} — {+1} is a coordinate function if there is some w € [W], 
such that f(x1,22,...,2w) = Xw; in other words, f = X{w}. 


DEFINITION 19.10 (LONG CODE) 
The long code for [W] encodes each w € [W] by the table of all values of the function x1} : 


EY | {41}. 


REMARK 19.11 
Note that w, normally written using log W bits, is being represented using a table of 2W bits, a 
doubly exponential blowup! This inefficiency is the reason for calling the code “long.” 


Similar to the test for the Walsh-Hadamard code, when testing the long code, we are given a 
function fai” — {+1}, and want to find out if f has good agreement with X{w} for some w, 


namely, faa is significant. Such a test is described in Exercise 16 of the previous chapter, but it 
is not sufficient for the proof of Hastad’s Theorem, which requires a test using only three queries. 
Below we show such a three query test albeit at the expense of achieving the following weaker 
guarantee: if the test passes with high probability then f has a good agreement with a function 
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Xa With |a| small (but not necessarily equal to 1). This weaker conclusion will be sufficient in the 
proof of Theorem 19.5. 

Let p > 0 be some arbitrarily small constant. The test picks two uniformly random vectors 
x,y € {+1} and then a vector z € {£1}! W] according to the following distribution: for every 
coordinate i € [W], with probability 1 — p we choose z; = +1 and with probability p we choose 
zi = —1. Thus with high probability, about p fraction of coordinates in z are —1 and the other 
1 — p fraction are +1. We think of z as a “noise” vector. The test accepts iff f(x) f(y) = f(xyz). 
Note that the test is similar to the linearity test except for the use of the noise vector z. 

Suppose f = Xs}. Then 


F(x) Fy) f (xyz) = LwYw(LwYwew) = 1+ zw 


Hence the test accepts iff zw = 1 which happens with probability 1 — p. We now prove a certain 
converse: 


LEMMA 19.12 N 
If the test accepts with probability 1/2 + € then Y, f3(1 — 2p)lel > Qe. 


PROOF: If the test accepts with probability 1/2 + e then E[ f(x) f(y) f(xyz)| = 2e. Replacing f by 
its Fourier expansion, we have 


2e < Exa [O faxal3)) O faxaly)) : O fx (xy2)) 
a B T 


= Exyz y fatetrxa Dx Nr ly) xr (2) 


a,Byy 


= Y fatal Enya xd NARA. 


By 


Orthonormality implies the expectation is 0 unless a = P = y, so this is 
= > fèEz[Xa(2)] 
Q 


Now Ez[Xa(z)] = Ez [[[uca 20) which is equal to J] 
dinate of z is chosen independently. Hence we get that 


2e< YRA- 20) 


wea Flw] = (1 — 2p)l°l because each coor- 


The conclusion of Lemma 19.12 is reminiscent of the calculation in the proof of Theorem 19.9, 
except for the extra factor (1 — 2p)lal, This factor depresses the contribution of fa for large a, 
allowing us to conclude that the small a’s must contribute a lot. This formalized in the following 
corollary (left as Exercise 2). 
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COROLLARY 19.13 
If f passes the long code test with probability 1/2+ 6 then 


> ee 


a:ja|<k 


where k = F log L, 


19.4 Proof of Theorem 19.5 


Recall that our proof of the PCP Theorem implies that there are constants y > 0,s € N such that 
(1+y)-GAP 2CSP, is NP-hard (see Claim 18.36). This means that for every NP-language L we have 
a PCP-verifier for L making two queries over alphabet {0,...,s—1} with perfect completeness and 
soundness parameter 1— y. Furthermore this PCP system has the property that the verifier accepts 
the answer pair 21, 22 iff z2 = h,;(z1) where hy is a function (depending on the verifier’s randomness 
r) mapping {0,...,s — 1} to itself (see Exercise 3). We call this the projection property. Using 
the Raz’s parallel repetition lemma (Theorem 19.3), we can reduce the soundness parameter to an 
arbitrary small constant at the expense of increasing the alphabet. Note that parallel repetition 
preserves the projection property. 

Let L be an NP-language and e > 0 an arbitrarily small constant. By the above there exists a 
constant W and PCP-verifier Vga, (having the projection property) that makes two queries to a 
polynomial-sized PCP proof 7 with alphabet {1,...,W} such that for every x, if z € L then there 
exists m such that Pr[Vf, (uv) = 1] = 1 and if z ¢ L then Pr[Vf, (xv) = 1] < e for every 7. 

Now we describe Hastad’s verifier Vy. It essentially follows Vraz, but it expects each entry in 
the PCP proof m to be encoded using the long code. It expects these encodings to be bifolded, 
a technical property we now define and is motivated by the observation that coordinate functions 
satisfy X(wy (UU) = —X(wy(U), where —u is the vector (—ux,..., —uyw). 


DEFINITION 19.14 
A function f : {+1} — (+1) is bifolded if for all u € {41}, f(—u) = —f(u). 


Whenever the PCP proof is supposed to contain a longcode codeword then we may assume 
without loss of generality that the function is bifolded. The reason is that the verifier can identify, 
for each pair of inputs u, —u, one designated representative —say the one whose first coordinate is 
+1— and just define f(—u) to be — f(u). One benefit —though of no consequence in the proof— 
of this convention is that bifolded functions require only half as many bits to represent. We will 
use the following fact: 


LEMMA 19.15 i 
If f : {41} — {41} is bifolded and fẹ 4 0 then |a| must be an odd number (and in particular, 
nonzero). 


PROOF: By definition, 
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If |a| is even then [[¿¿. ui = [ica (—ui). So if f is bifolded, the terms corresponding to u and —u 
have opposite signs and the entire sum is 0. W 


Hastad’s verifier. Recall that VRaz uses its randomness to select a function two entries i,j in 
the table 7 and a function h : [W] — [W], and accepts iff r(j) = h(r(i)). Hastad’s verifier, 
denoted Vz, expects the proof 7 to consist of (bifolded) longcode encodings of each entry of m. The 
verifier Vz emulates Vraz to pick two locations i,j in the table and a function h : [W] — [W] such 
that Vraz's test is to accept iff [3] = h(r[i]). The proof 7 contains in the locations i and j two 
functions f and g respectively (which may or may not be the longcode encoding of r(¿) and r(j)). 
Instead of reading the long codes f,g in their entirety, the verifier Vy performs a simple test that 
is reminiscent of the long code test. For a string y € {+1} we denote by h~!(y) the string such 
that for every w € [W], h-'(y)wy = Yn(w): In other words, for each u € [W], the bit y, appears in 
all coordinates of h~!(y) that are indexed by integers in the subset h7*(u). This is well defined 
because [h7*(u) : u € [W]} is a partition of [W]. Vy chooses uniformly at random u, y € {+1}” 


and chooses z € {+1}™ by letting zi = +1 with probability 1 — p and z; = —1 with probability p. 
It then accepts Iff 
fugly) = f(h*(y)uz) (1) 
Translating back from {+1} to {0,1}, note that Vp’s test is indeed linear, as it accepts iff 
Tlia] + Tliz] + [ig] = b for some i1,12,13 € [m2W] and b € {0,1}. (The bit b can indeed equal 1 
because of the way Vy ensures the bifolding property.) 


Completeness of Vz. Suppose f,g are long codes of two integers w, u satisfying h(w) = u (in 
other words, Va. would have accepted the assignments represented by these integers). Then 


FI) (y)uz) = uwyu(h (y)uzu 
= Uv Yu(Yr(w) Uw2w) = 2w. 
Hence Vy accepts iff zw = 1, which happens with probability 1 — p. 


Soundness of Vy. We now show that if Vy accepts f,g with probability significantly more than 
1/2, then the Fourier transforms of f, g must be correlated. To formalize this we define for a € [W], 


hala) = {ue [W]: |h7*(u) nal is odd} 
Notice in particular that for every u € ha(a) there is at least one w € a such that h(w) = u. 


In the next Lemma 6 is allowed to be negative. 


LEMMA 19.16 
Let f,g : {41}" 5 {41}, h : [W] — [W] be bifolded functions passing Vy's test (1) with 
probability at least 1/2+ 6. Then 


Y Fina (1 — 2p)!! > 26 
aC[W],a40 
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PROOF: By hypothesis, f,g are such that E[f(u)f(uh—'(y)z)g(y)] > 26. Replace f,g by their 
Fourier expansions. We get that 


25 < = Eu yz | C faxa) (S davaly) (D> fyxq(uh-"(y)2)) 
Q B Y 


= X fade, Enya [Xa lU) xey) xU) xh y) x (2)] 
a, b,y 


By orthonormality this simplifies to 


= X f2GaEy.2 lxoly)xalh” '(y))xa(2)] 
0,8 


= $ faĝo l — 20) Ey [xa(h y) xa) (2) 
ab 


since Xa(z) = (1 — 2p)!!, as noted in our analysis of the long code test. Now we have 


Ey[xa(h*(y))xa(y)] = Eyl] [Mw [1 vel 


wea ues 
= Eyl] Yh(w) II Yul, 
wea ucl 


which is 1 if ho(a) = 8 and O otherwise. Hence (2) simplifies to 


y F2.Gno(a) (1 — 2p). 


Finally we note that since the functions are assumed to be bifolded, the Fourier coefficients fo and 
gg are zero. Thus those terms can be dropped from the summation and the Lemma is proved. W 


The following corollary of Lemma 19.16 completes the proof of Hastad’s 3-bit PCP Theorem. 


COROLLARY 19.17 
Let e be the soundness parameter of Vraz. If p,6 satisfy pd? > e then the soundness parameter of 
Vy is at most 1/2 + ô. 


PROOF: Suppose Vy accepts a proof 7 with probability at least 1/2 + ô. We give a probabilistic 
construction of a proof 7 causing Vraz to accept the same statement with probability at least pd. 

Suppose that Vea. uses proofs m with m entries in [W]. We can think of 7 as providing, for 
every i € [m], a function f; : {£1} {41}. We will use 7 to construct a proof 7 for Vraz as follows: 
we first use f; to come up with a distribution D; over [W]. We then let z[i] be a random element 


from D;. 
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The distribution D;. Let f = f;. The distribution D; is defined by first selecting a C [W] with 
probability E and then selecting w at random from a. This is well defined because >”, Ë = 1 and 
(due to bifolding) fø = 0. 

Recall that Vraz picks using its random tape a pair i, j of locations and a function h : [W] — [W] 
and then verifies that 7[j] = h(z[7]). Let r be some possible random tape of Vraz and let i, j, h be 
the pair of entries in 7 and function that are determined by r. We define the indicator random 
variable I, to be 1 if for w €r Di and u Er Dj it holds that w = h(u) and to be 0 otherwise. Thus, 
our goal is to show that 

Ex=D1,....Dm (Erlr]] = po” (3) 
since that would imply that there exists a table 7 causing VRaz to accept with probability at least 
po, proving the corollary. 

To prove (3) we first notice that linearity of expectation allows us to exchange the order of the 
two expectations and so it is enough to bound E,[Ep,,p,[J,]] where i,j are the entries determined 
by the random tape r. For every r denote by 6, the probability that Vy accepts 7 when it uses r 
as the random tape for Vraz. The acceptance probability of Vz is E-[5 + ôr] and hence E,[6,] = ô. 

Let 2,7,h be the pair and function determined by r and denote by f = f; and g = fj where 
fi (resp. fj) is the function at the it? (resp. jt) entry of the table 7. What is the chance that a 
pair of assignments w Er D; and v Er Dj will satisfy the constraint? (i.e., will satisfy v = h(w)?). 
Recall that we pick w and u by choosing a with probability Jz G6 with probability 43 and choosing 
w ER a,v Er B. Now if 8 = ha(a) then for every v € 8 there exists w € a with h(w) = v and 
hence the probability the constraint is satisfied is at least 1/|a|. Thus, we have that 


ls, 
y Ja] 2a) Ss ED;,D; [Z] (4) 


This is similar to (but not quite the same as) the expression in Lemma 19.16, according to 
which 


28, < X mata) (1 — 2p). 


we have 


However, since one can easily see that (1 — 2p)!¢l < 


2 
v p|a| 


20, < ) fa lónato) laa 


SVP SD fa lôn 7 


Applying the Cauchy-Schwartz inequality, >>, a;b; < O >; aa. bee with fa [dro (a | Ja 


playing the role of the a;’s and fa playing that of the b;’s, we obtain 


1/2 1/2 
VP < 2 fè lnl aj S (22) (Se Thala 1) (5) 
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Since >, f2 = 1, by squaring (5) and combining it with (4) we get that for every r, 
5; p < Ep,,D; [£,] 
taking expectation over r and using E[X]? < E[X?] we get that 
dp = Er [5,]?p < Er[67]p < Er[Ep;,p, Url] 


proving (3). Ml 


19.5 Learning Fourier Coefficients 


Suppose that you are given random access to a Boolean function f : {+1}" — {+1} and want to 
find the high Fourier coefficients of f. Of course, we can compute all of the coefficients in time 
polynomial in 2”, but is there a faster algorithm? By the Parseval equality (Lemma 19.7) we know 
that there can be at most 1/e? coefficients with absolute value larger than e, and so we can hope 
to learn these coefficients in time polynomial in n, and 1/e. It turns out we can (almost) achieve 
this goal: 


THEOREM 19.18 ([?]) 

There is an algorithm A that given input n € N,e € (0,1) and random access to a function 
f : {+1}" — {+1}, runs in poly(n,1/e) time and with probability at least 0.9 outputs a set L of 
size at most O(1/e2) such that for every a C [n], if |fa| > e then a € L. 


PROOF: We identify subsets of [n] with strings in {0,1} in the obvious way. For k < n and 


a € {0,1}* denote 
fax = > Top 
Be{0,1}"-* 


where o denotes concatenation. By Parseval (Lemma 19.7) f, = 1. Note also that for every k < n 
and a € {0,1}*, fas = faor + fais. Therefore, if we think of the full depth-n binary labeled by 
binary strings of length < n (with the root being the empty word and the two children of a are a0 
and a1), then at any level of this tree there can be at most 1/e? strings a such that fa, > €? (the 
kt” level of the tree corresponds to all strings of length k). Note that if a string a satisfies fee 
then the same holds for every string of the form ao 6. Our goal will be to find all these strings at 
all levels, and then output all the strings that label leaves in the tree (i.e., all n-bit strings). 

The heart of the algorithm is a procedure Estimate that given a and oracle access to f(-), 
outputs an estimate of fy within e/4 accuracy with probability 1 — i, Using this procedure we 
work our way from the root down, and whenever Estimate(a) gives a value smaller than e/2 we 
“kill” this node and will not deal with it and its subnodes. Note that unless the output of Estimate 
is more than e/4-far from the real value (which we will ensure by the union bound happens with 
probability less than 0.1 over all the levels) at most 4/e nodes will survive at any level. The 
algorithm will output the 4/€ leaves that survive. 

The procedure Estimate uses the following claim: 
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CLAIM 19.19 
For every a, 


for = Exento yento eLo Y) F(X! O¥)Xalx)Xal(X’)] 


PROOF: We start with the case that a = 0%. To get some intuition, suppose that Foi , =l. This 
means that f can be expressed as a sum of functions of the form Xorog and hence it does not depend 
on its first k variables. Thus f(xoy) = f(x’oy) and we'll get that E| f (xoy) f(x’oy)] = E[f(z)?] = 1. 
More generally, if for , is large then that means that in the Fourier representation, the weight of 
functions not depending on the first k variables is large and hence we expect large correlation 
between f(x’ o y) and f(xoy). This is verified by the following calculations: 


—n—k t = 
2 2, PESAS a 
XxX, X DA 


2 Y (Y yo B)xyoa(xoy) | | Y yo PB xo oy) = 


7 Xyop (x 0 y) = xy (x)xp8 (y) 
xx y (18 y 


of! 
22H SES Fly e B)xy(«)xa(y) Ho BY xy (xa (y) = 


x, x’ y \ yol y'op' 
Y PODRA ER o Xy (x x) + (E Ge) 2" pue. = o 
yB B x! y E CE mA SOR A 
F(0* o B) f(0* o Boag = X FO Sra 
pp B 


For the case a 4 0", we essentially add these factors to translate it to the case a = 0%. Indeed 
one can verify that if we define g(xoy) = f(xoy)xa(x) then for every 3 € {0,1}"*. Jokog = faop- 
a 


By the Chernoff bound, we can estimate the expectation of Claim 19.19 (and hence Fon) using 
repeated sampling, thus obtaining the procedure Estimate and completing the proof. M 


19.6 Other PCP Theorems: A Survey 


The following variants of the PCP Theorem have been obtained and used for various applications. 


19.6.1 PCP’s with sub-constant soundness parameter. 


Because ¢-times parallel repetition transforms a proof of size m to a proof of size mf, 


use it with £ larger than a constant and still have a polynomial-sized proof. Fortunately, there have 
been direct constructions of PCP’s achieving low soundness using larger alphabet size, but without 
increasing the proof’s size. Raz and Safra [?] show that there is an absolute constant q such that 


we cannot 
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for every W < ylog n, every NP language has a q-query verifier over alphabet {0,...,W — 1} that 


uses O(log n) random bits, and has soundness 2 log W), 


19.6.2 Amortized query complexity. 


Some applications require binary-alphabet PCP systems enjoying a tight relation between the 
number of queries (that can be an arbitrarily large constant) and the soundness parameter. The 
relevant parameter here turns out to be the free bit complexity [?, ?]. This parameter is defined as 
follows. Suppose the number of queries is q. After the verifier has picked its random string, and 
picked a sequence of q addresses, there are 2% possible sequences of bits that could be contained 
in those addresses. If the verifier accepts for only t of those sequences, then we say that the 
free bit parameter is logt (note that this number need not be an integer). In fact, for most 
applications it suffices to consider the amortized free bit complexity [?]. This parameter is defined 
as lims 50 fs/log(1/s), where fs is the number of free bits needed by the verifier to ensure the 
soundness parameter is at most s. Hástad constructed systems with amortized free bit complexity 
tending to zero [?]. That is, for every e > 0, he gave a PCP-verifier for NP that uses O(log n) 
random bits and e amortized free bits. He then used this PCP system to show (using tools 
from [?, ?, ?]) that MAX INDSET (and so, equivalently, MAX CLIQUE) is NP-hard to approximate 
within a factor of n!7* for arbitrarily small e > 0. 


19.6.3 Unique games. 
Exercises 


$1 Prove that there is a polynomial-time algorithm that given a satisfiable 2CSPy instance y 
over {0..W—I} where all the constraints are permutations (i.e, p; checks that uj = h(u;) for 
some j, j’ € [n] and permutation h : {0..W—l} — {0..W—1}) finds a satisfying assignment u 
for y. 


§2 Prove Corollary 19.13. 


§3 Prove that the PCP system resulting from the proof of Claim 18.36 (Chapter 18) satisfies 
the projection property. 


$4 Let f : {41}" — {+1} and let I C [n]. Let M; be the following distribution: we choose 
z Er Mq by for i € I, choose z; to be +1 with probability 1/2 and —1 with probability 1/2 
(independently of other choices), for i ¢ I choose z; = +1. We define the variation of f on I 
to be Pryep{+1}",2e pM lf (x) A f(x2)]. 


Suppose that the variation of f on J is less than e. Prove that there exists a function 
g : {+1}”" — R such that (1) g does not depend on the coordinates in J and (2) g is 10e-close 
to f (ie, Prxeg{+1}” [f (x) 4 g(x)] < 10e). Can you come up with such a g that outputs 
values in {+1} only? 


85 For f : {£1}" — {+1} and x € {+1}” we define Ny(x) to be the number of coordinates i 
such that if we let y to be x flipped at the it” coordinate (i.e., y = ze’ where et has —1 in the 
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qth 


coordinate and +1 everywhere else) then f(x) 4 f(y). We define the average sensitivity 
of f, denoted by as( f) to be the expectation of Ny(x) for x Er {+1}". 


(a) Prove that for every balanced function f : (+1)” — {+1} (ie., Pr[ f(x) = +1] = 1/2), 


as(f) > 1. 
(b) Let f be balanced function from (+1)” to (+1) with as(f) = 1. Prove that f is a 
coordinate function or its negation (i.e., f(x) = x; or f(x) = —x; for some i € [n] and 


for every x € {+1}”). 


Chapter 20 


Quantum Computation 


“Turning to quantum mechanics.... secret, secret, close the doors! we always have 
had a great deal of difficulty in understanding the world view that quantum mechanics 
represents ... It has not yet become obvious to me that there’s no real problem. I 
cannot define the real problem, therefore I suspect there’s no real problem, but I’m 
not sure there’s no real problem. So that’s why I like to investigate things.” 
Richard Feynman 1964 


“The only difference between a probabilistic classical world and the equations of the 
quantum world is that somehow or other it appears as if the probabilities would have 
to go negative..” 

Richard Feynman, in “Simulating physics with computers”, 1982 


Quantum computers are a new computational model that may be physically realizable and 
may have an exponential advantage over ‘classical” computational models such as probabilistic 
and deterministic Turing machines. In this chapter we survey the basic principles of quantum 
computation and some of the important algorithms in this model. 

As complexity theorists, the main reason to study quantum computers is that they pose a 
serious challenge to the strong Church-Turing thesis that stipulates that any physically reasonable 
computation device can be simulated by a Turing machine with polynomial overhead. Quantum 
computers seem to violate no fundamental laws of physics and yet currently we do not know any 
such simulation. In fact, there is some evidence to the contrary: as we will see in Section 20.7, 
there is a polynomial-time algorithm for quantum computers to factor integers, where despite 
much effort no such algorithm is known for deterministic or probabilistic Turing machines. In fact, 
the conjectured hardness of this problem underlies of several cryptographic schemes (such as the 
RSA cryptosystem) that are currently widely used for electronic commerce and other applications. 
Physicists are also interested in quantum computers as studying them may shed light on quantum 
mechanics, a theory which, despite its great success in predicting experiments, is still not fully 
understood. 

This chapter utilizes some basic facts of linear algebra, and the space C”. These are reviewed in 
Appendix A. See also Note 20.8 for a quick reminder of our notations. 
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20.1 Quantum weirdness 


It is beyond this book (and its authors) to fully survey quantum mechanics. Fortunately, only 
very little physics is needed to understand the main results of quantum computing. However, 
these results do use some of the more counterintuitive notions of quantum mechanics such as the 
following: 


Any object in the universe, whether it is a particle or a cat, does not have definite 
properties (such as location, state, etc..) but rather has a kind of probability wave over 
its potential properties. The object only achieves a definite property when it is observed, 
at which point we say that the probability wave collapses to a single value. 


At first this may seem like philosophical pontification analogous to questions such as “if a tree 
falls and no one hears, does it make a sound?” but these probability waves are in fact very real, in 
the sense that they can interact and interfere with one another, creating experimentally measurable 
effects. Below we describe two of the experiments that led physicists to accept this counterintuitive 
theory. 


20.1.1 The 2-slit experiment 


In the 2-slit experiment a source that fires electrons one by one (say, at the rate of one electron per 
second) is placed in front of a wall containing two tiny slits (see Figure ??). Beyond the wall we 
place an array of detectors that light up whenever an electron hits them. We measure the number 
of times each detector lights up during an hour. 

When we cover one of the slits, we would expect that the detectors that are directly behind the 
open slit will receive the largest number of hits, and as Figure ?? shows, this is indeed the case. 
When both slits are uncovered we expect that the number of times each detector is hit is the sum 
of the number of times it is hit when the first slit is open and the number of times it is hit when the 
second slit is open. In particular, uncovering both slits should only increase the number of times 
each location is hit. 

Surprisingly, this is not what happens. The pattern of hits exhibits the “interference” phenom- 
ena depicted in Figure ??. In particular, at several detectors the total electron flux is lower when 
both slits are open as compared to when a single slit is open. This defies explanation if electrons 
behave as particles or “little balls”. 

According to Quantum mechanics, the explanation is that it is wrong to think of an electron has 
a “little ball” that can either go through the first slit or the second (i.e., has a definite property). 
Rather, somehow the electron instantaneously explores all possible paths to the detectors (and so 
“finds out” how many slits are open) and then decides on a distribution among the possible paths 
that it will take. 

You might be curious to see this “path exploration” in action, and so place a detector at each 
slit that light up whenever an electron passes through that slit. When this is done, one can see 
that every electron passes through only one of the slits like a nice little ball. But furthermore, the 
interference phenomenon disappears and the graph of electron hits becomes, as originally expected, 
the sum of the hits when each slit is open. The explanation is that, as stated above, observing an 
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NOTE 20.1 (PHYSICALLY IMPLEMENTING QUANTUM COMPUTERS.) 


object “collapses” their distribution of possibilities, and so changes the result of the experiment. 
(One moral to draw from this is that quantum computers, if they are ever built, will have to be 
carefully isolated from external influences and noise, since noise may be viewed as a “measurement” 
performed by the environment on the system. Of course, we can never completely isolate the system, 
which means we have to make quantum computation tolerant of a little noise. This seems to be 
possible under some noise models, see the chapter notes.) 


20.1.2 Quantum entanglement and the Bell inequalities. 


Even after seeing the results of the 2-slit experiment, you might still be quite skeptical of the 
explanation that quantum mechanics offers. If you do, you are in excellent company. Albert 
Einstein didn’t buy it either. While he agreed that the 2-slit experiment means that electrons 
are not exactly “little balls”, he didn’t think that it is sufficient reason to give up such basic 
notions of physics such as the existence of an independent reality, with objects having definite 
properties that do not depend on whether one is observing them. To show the dangerous outcomes 
of giving up such notions, in a 1951 paper with Podosky and Rosen (EPR for short) he described a 
thought experiment showing that accepting Quantum mechanics leads to the seemingly completely 
ridiculous conclusion that systems in two far corners of the universe can instantaneously coordinate 
their actions. 

In 1964 John Bell showed how the principles behind EPR thought experiment can be turned into 
an actual experiment. In the years since, Bell’s experiment has been repeated again and again with 
the same results: quantum mechanics’ predictions were verified and, contrary to Einstein’s expec- 
tations, the experiments refuted his intuitions about how the universe operates. In an interesting 
twist, in recent years the ideas behind EPR’s and Bell’s experiments were used for a practical goal: 
encryption schemes whose security depends only on the principles of quantum mechanics, rather 
than any unproven conjectures such as P 4 NP. 

For complexity theorists, probably the best way to understand Bell’s experiment is as a two 
prover game. Recall that in the two prover setting, two provers are allowed to decide on a strategy 
and then interact separately with a polynomial-time verifier which then decides whether to accept 
or reject the interaction (see Chapters 8 and 18). The provers’ strategy can involve arbitrary 
computation and even be randomized, with the only constraint being that the provers are not 
allowed to communicate during their interaction with the verifier. 


Bell’s game. In Bell’s setting, we have an extremely simple interaction between the verifier and 
two provers (that we’ll name Alice and Bob): there is no statement that is being proven, and all the 
communication involves the verifier sending and receiving one bit from each prover. The protocol 
is as follows: 
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1. The verifier chooses two random bits x, y Er {0,1}. 
2. It sends x to Alice and y to Bob. 

3. Let a denote Alice’s answer and b Bob’s answer. 

4. The verifier accepts if and only if a b= zx ^y. 


It is easy for Alice and Bob to ensure the verifier accepts with probability 3/4 (e.g., by always 
sending a = b= 0). It turns out this is the best they can do: 


THEOREM 20.2 ([BEL64])) 
Regardless of the strategy the provers use, the verifier will accept with probability at most 3/4. 


PROOF: Assume for the sake of contradiction that there is a (possibly probabilistic) strategy that 
causes the verifier to accept with probability more than 3/4. By a standard averaging argument 
there is a fixed set of provers’ coins (and hence a deterministic strategy) that causes the verifier to 
accept with at least the same probability, and hence we may assume without loss of generality that 
the provers’ strategy is deterministic. 

A deterministic strategy for the two provers is a pair of functions f,g : {0,1} — {0,1} such as 
the provers’ answers a,b are computed as a = f(x) and b = g(y). The function f can be one of only 
four possible functions: it can be either the constant function zero or one, the function f(x) = x 
or the function f(y) =1-— y. We analyze the case that f(x) = x; the other case are similar. 

If f(x) = x then the verifier accepts iff b = (x A y) x. On input y, Bob needs to find b that 
makes the verifier accept. If y = 1 then Ay = x and hence b = 0 will ensure the verifier accepts 
with probability 1. However, if y = 0 then (x A y) 6 x = x and since Bob does not know zx, the 
probability that his output b is equal to x is at most 1/2. Thus the total acceptance probability is 
at most 3/4. W 


What does this game have to do with quantum mechanics? The main point is that according 
to “classical” pre-quantum physics, it is possible to ensure that Alice and Bob are isolated from 
one another. Suppose that you are given a pair of boxes that implement some arbitrary strategy 
for Bell’s game. How can you ensure that these boxes don’t have some secret communication 
mechanism that allows them to coordinate their answers? We might try to enclose the devices 
in lead boxes, but even this does not ensure complete isolation. However, Einstein’s theory of 
relativity allows us a foolproof way to ensure complete isolation: place the two devices very far 
apart (say at a distance of a 1000 miles from one another), and perform the interaction with each 
prover at a breakneck speed: toss each of the coins x,y and demand the answer within less than 
one millisecond. Since according to the theory of relativity, nothing travels faster than light (that 
only covers about 200 miles in a millisecond), there is no way for the provers to communicate and 
coordinate their answers, no matter what is inside the box. 

The upshot is that if someone provides you with such devices that consistently succeed in this 
experiment with more than 3/4 = 0.75 probability, then she has refuted Einstein’s theory. As we 
will see later in Section 20.3.2, quantum mechanics, with its instantaneous effects of measurements, 
can be used to actually build devices that succeed in this game with probability at least 0.8 (there 
are other games with more dramatic differences of probabilities) and this has been experimentally 
demonstrated. 
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20.2 A new view of probabilistic computation. 


To understand quantum computation, it is helpful to first take a different viewpoint of a process 
we are already familiar with: probabilistic computation. 

Suppose that we have an m-bit register. Normally, we think of such a register as having 
some definite value x € {0,1}’". However, in the context of probabilistic computation, we can 
also think of the register’s state as actually being a probability distribution over its possible val- 
ues. That is, we think of the register’s state as being described by a 2’-dimensional vector 
v = (Vom,Vom-11,..., Vim), where for every x € {0,1}"", vx € [0,1] and >>, vz = 1. When 
we read, or measure, the register, we will obtain the value x with probability vz. 

For every x € {0,1}”, we denote by |x) the vector that corresponds to the degenerate distribu- 
tion that takes the value x with probability 1. That is, |x} is the 2-dimensional vector that has 
zeroes in all the coordinates except a single 1 in the x*” coordinate. Note that v = reto 1y” Va (2). 
(We think of all these vectors as column vectors in the space R™.) 


EXAMPLE 20.3 
If a 1-bit register’s state is the distribution that takes the value 0 with probability p and 1 with 
probability 1 — p, then we describe the state as the vector p|0) + (1 — p) |1). 
The uniform distribution over the possible values of a 2-bit register is represented by 1/4 (|00) + |01) + |10) + |11 
The distribution that is uniform on every individual bit, but always satisfies that the two bits are 
equal is represented by 1/2 (|00) + |11)). 


An probabilistic operation on the register involves reading its value, and, based on the value 
read, modifying it in some deterministic or probabilistic way. If F is some probabilistic operation, 
then we can think of F as a function from R?” to R?” that maps the previous state of the register 
to its new state after the operation is performed. There are certain properties that every such 
operation must satisfy: 


e Since F depends only on the contents of the register, and not on the overall distribution, for 
every v, F(v) = Dre {o,1}" VF (|v). That is, F is a linear function. (Note that this means 
that F can be described by a 2” x 2” matrix.) 


e If v isa distribution vector (i.e., a vector of non-negative entries that sum up to one), then so 
is F(v). That is, F is a stochastic function. (Note that this means that viewed as a matrix, 
F has non-negative entries with each column summing up to 1.) 


EXAMPLE 20.4 

The operation that, regardless of the register’s value, writes into it a uniformly chosen random 
string, is described by the function F such that F(lx)) =2""> oe(o1y» |£) for every x € (0, es 
(Because the set {|7)},c¢9,1} is a basis for R?2”, a linear function is completely determined by its 
output on these vectors.) 
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The operation that flips the first bit of the register is described by the function F such that 
F(|t1...2m)) = |(1 — £1)@2...2%m) for every £1,..., Tm E {0,1}. 


Of course there are many probabilistic operations that cannot be efficiently computed, but there 
are very simple operations that can certainly be computed. An operation F is elementary if it only 
reads and modifies at most three bits of the register, leaving the rest of the register untouched. 
That is, there is some operation G : R?” — R and three indices j,k, € [m] such that for every 
Dli=oy Em € {0,1}, F(|£1...£m)) = ly... Ya) where |yjykye) = G(|£j£kze)) and yi = x; for 
every i € {j,k,¢}. Note that such an operation can be represented by a 2? x 2? matrix and three 
indices in [m]. 


EXAMPLE 20.5 
Here are some examples for operations depending on at most three bits: 


AND function Coin Tossing Constant zero function 
Flxyz> = |xy(xAy)> | Fix> = 1/2|0>+1/2|1> F> = |0> 
000 001 010 011 100 101 110 111 
oo/1 1 00 0 0 0 
001 0 1 o 1 


0/1/2 1/2 oli 1 
1\1/2 1/2 110 ,) 


For example, if we apply the coin tossing operation to the second bit of the register, then 
this means that for every z = z1 ... Zm € {0,1}”, the vector |z} is mapped to 1/2|z10z3... zm} + 
Valzlz3... Zm}. 


E 
Sessa 
sebas es 
Sais 
EEN 
aisaen 
saaks a 
soe 
dae gaoa 


We define a probabilistic computation to be a sequence of such elementary operations applied 
one after the other (see Definition 20.6 below). We will later see this corresponds exactly to our 
previous definition of probabilistic computation as in the class BPP defined in Chapter 7. 
DEFINITION 20.6 (PROBABILISTIC COMPUTATION) 

Let f : {0,1}* — {0,1}* and T : N — N be some functions. We say that f is computable in 
probabilistic T(n)-time if for every n € N and x € {0,1}", f(x) can be computed by the following 
process: 


1. Initialize an m bit register to the state |z0"~™ ) (1.e., x padded with zeroes), where m < T(n). 


2. Apply one after the other T(n) elementary operations F\,...,F 7 to the register (where we 
require that there is a polynomial-time TM that on input 1”, 17) outputs the descriptions 
of Fiy .. , Er). 


3. Measure the register and let Y denote the obtained value. (That is, if v is the final state of 
the register, then Y is a random variable that takes the value y with probability v, for every 


y € {0,1}".) 
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Denoting @ = |f(x)|, we require that the first £ bits of Y are equal to f(x) with probability at 
least 2/3. 


PROBABILISTIC COMPUTATION: SUMMARY OF NOTATIONS. 
The state of an m-bit register is represented by a vector v € R?” such that 
the register takes the value x with probability vz. 
An operation on the register is a function F : R?” — R?” that is linear and 
stochastic. 
An elementary operation only reads and modifies at most three bits of the 
register. 
A computation of a function f on input x € {0,1}" involves initializing the 
register to the state |z0™~”), applying a sequence of elementary operations 
to it, and then measuring its value. 
Now, as promised, we show that our new notion of probabilistic computation is equivalent to 
the one encountered in Chapter 7. 


THEOREM 20.7 
A Boolean function f : {0,1}" — {0,1} is in BPP iff it is computable in probabilistic p(n)-time 
for some polynomial p : N > N. 


PROOF: (=) Suppose that f € BPP. As we saw before (e.g., in the proof of Theorem 6.7) this 
means that f can be computed by a polynomial-sized Boolean circuit C (that can be found by 
a deterministic poly-time TM) if we allow the circuit C access to random coins. Thus we can 
compute f as follows: we will use a register of n+r + s bits, where r is the number of random coins 
C uses, and s is the number of gates C uses. That is, we have a location in the register for every 
coin and every gate of C. The elementary coin tossing operation (see Example 20.5) can transform 
a location initialized to 0 into a random coin. In addition, we have an elementary operation that 
transforms three bits x,y and z into x,y,x A y and can similarly define elementary operations for 
the OR and NOT functions. Thus, we can use these operations to ensure that for every gate of C, 
the corresponding location in the register contains the result of applying this gate when the circuit 
is evaluated on input z. 

(<) We will show a probabilistic polynomial-time algorithm to execute an elementary operation 
on a register. To simulate a p(n)-time probabilistic computation we can execute this algorithm 
p(n) times one after the other. For concreteness, suppose that we need to execute an operation on 
the first three bits of the register, that is specified by an 8 x 8 matrix A. The algorithm will read 
the three bits to obtain the value z € {0, qe. and then write to them a value chosen according to 
the distribution specified in the zt” column of A. 

The only issue remaining is how to pick a value from an arbitrary distribution (p1,...,pg) over 
{0,1}? (which we identify with the set [8]). One case is simple: suppose that for every i € [8], 
pi = K;/2° where £ is polynomial in £ and K,,...,Kg € [2%]. In this case, the algorithm will 
choose using £ random coins a number X between 1 and 2° and output the largest i such that 
xe ak 

However, this essentially captures general case as well: every number p € [0,1] can be ap- 
proximated by a number of the form K/2‘ within 27! accuracy. This means that every general 
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NOTE 20.8 (A FEW NOTIONS FROM LINEAR ALGEBRA) 
We use in this chapter several elementary facts and notations involving the 
space CM. These are reviewed in Appendix A, but here is a quick reminder: 


e If z = a + ib is a complex number (where i = y—1), then Z = a — ib 
denotes the complex conjugate of z. Note that zz = a? + b? = |z|?. 


e The inner product of two vectors u, v € C™, denoted by (u, v}, is equal 
to rem] Ur Va 


e The norm of a vector u, denoted by |lul|,, is equal to \/(u,u) = 


4/2 zejm] ju,,|?. 


e If (u,v) = 0 we say that u and v are orthogonal. More generally, 
Ou, v = cos 4||ul|, ||v||,, where 8 is the angle between the two vectors 
u and v. 


e If Ais an M x M matrix, then A’ denotes the conjugate transpose of 
A. That is, Aly = Ay» for every x,y € [M]. 


e An Mx M matrix A is unitary if AAt = I, where I is the M x M 
identity matrix. 


Note that if z is a real number (i.e., z has no imaginary component) then 
z = z. Hence, if all vectors and matrices involved are real then the inner 
product is equal to the standard inner product of R” and the conjugate 
transpose operation is equal to the standard transpose operation. Also a 
real matrix is unitary if and only if it is symmetric. 


distribution can be well approximated by a distribution over the form above, and so by choosing a 
good enough approximation, we can simulate the probabilistic computation by a BPP algorithm. 
a 


20.3 Quantum superposition and the class BQP 


A quantum register is also composed of m bits, but in quantum parlance they are called “qubits”. 
In principle such a register can be implemented by any collection of m physical systems that can 
have an ON and OFF states, although in practice there are significant challenges for such an 
implementation (see Note 20.1). According to quantum mechanics, the state of such a register can 
be described by a 2’’-dimensional vector that, unlike the probabilistic case, can actually contain 
negative and even complex numbers. That is, the state of the register is described by a vector 
v € C?”. Once again, we denote v = dueto 1” Vo |Z) (where again |x) is the column vector that 


has all zeroes and a single one in the zt” coordinate). However, according to quantum mechanics, 
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when the register is measured, the probability that we see the value x is given not by vz but by 
|v..|?. This means that v has to be a unit vector, satisfying `, |v,|? = 1 (see Note 20.8). 


EXAMPLE 20.9 

The following are two legitimate state vectors for a two-qubit quantum register: A 10) + Y 11) 
1 
+ 
or 1 with probability 1/2, these are considered distinct states and we will see that it is possible to 


differentiate between them using quantum operations. 

Because states are always unit vectors, we often drop the normalization factor and so, say, use 
10) — |1) to denote the state a 10) — a \1). 

We call the state where all coefficients are equal the uniform state. For example, the uniform 
state for a 4-qubit register is 


and —= |0) — a |1). Even though in both cases, if the register is measured it will contain either 0 


|00) + |01) + |10) + |11), 


(where we dropped the normalization factor of 3.) We will also use the notation |x) y) to denote 
the standard basis vector |xy). It is easily verified that this operation respects the distributive law, 
and so we can also write the uniform state of a 4-qubit register as 


(10) +11)) 00) + |1)) 


Once again, we can view an operation applied to the register as a function F that maps its 
previous state to the new state. That is, F is a function from C?” to C?”. According to quantum 
mechanics, such an operation must satisfy the following conditions: 


1. F is a linear function. That is, for every v € C”, F(v) = 0, VsF (|£)). 


2. F maps unit vectors to unit vectors. That is, for every v with ||v||, = 1, ||F(v)||, = 1. 


Together, these two conditions imply that F can be described by a 2™ x 2 unitary matrix. 
That is, a matrix A satisfying AAt = I (see Note 20.8). We recall the following simple facts about 
unitary matrices (left as Exercise 1): 


CLAIM 20.11 

For every M x M complex matrix A, the following conditions are equivalent: 
1. A is unitary (i.e., AAt = I). 
2. For every vector v € CA, ||Av||, = ||w!|... 


3. For every orthonormal basis {v'} of C™ (see below), the set { Av’ } 


mal basis of CM, 


is an orthonor- 


iE[M] ¡e[M 


4. The columns of A form an orthonormal basis of CM, 
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NOTE 20.10 (THE GEOMETRY OF QUANTUM STATES) 

It is often helpful to think of quantum states geometrically as vectors in 
space. For example, consider a single qubit register, in which case the state is 
a unit vector in the two-dimensional plane spanned by the orthogonal vectors 
|0) and |1). For example, the state v = cos 0 |0} + sin 0 |1} corresponds to a 
vector making an angle 9 with the |0) vector and an angle 7/2 — 0 with the 
|1) vector. When v is measured it will yield 0 with probability cos? 9 and 1 
with probability sin? 0. 


|1> 


v = cos 9 |0> + sin 6 |1> 


sin 0 


0> 


cos O 


Although it’s harder to visualize states with complex coefficients or more 
than one qubit, geometric intuition can still be useful when reasoning about 
such states. 
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5. The rows of A form an orthonormal basis of C™. 


(Recall that a set lv iem of vectors in C™ is an orthonormal basis of C™ if for every i, j € [M], 


(vi, vi) is equal to 1 if i = j and equal to 0 if i 4 j, where (v,u) is the standard inner product 
over CM. That is, (v, u) = -i v¿U;.) 

As before, we define an elementary quantum operation to be an operation that only depends 
and modifies at most three qubits of the register. 


EXAMPLE 20.12 

Here are some examples for quantum operations depending on at most three qubits. (Because all 
the quantum operations are linear, it suffices to describe their behavior on any linear basis for the 
space C?” and so we often specify quantum operations by the way they map the standard basis.) 


e The standard NOT operation on a single bit can be thought of as the unitary operation that 
maps |0) to |1) and vice versa. 


e The Hadamard operation is the single qubit operation that (up to normalization) maps |0) 
to [0) +|1) and |1) to [0) —|1). (More succinctly, the state |b) is mapped to |0) + (-1)? |1).) 
It turns out to be a very useful operation in many algorithms for quantum computers. Note 


that if we apply an Hadamard operation to every qubit of an n-qubit register, then for every 
x € [0,1)”, the state |x} is mapped to 


(10) + (-D” 119100) + (-1) |1)) --- (10) + (-1)*" |1)) = 
Y miya DA= Y -1 ly), 


ye{0,1}” ye{0,1}” 


where x O y denotes the inner product modulo 2 of x and y. That is, O y = Oy, wiyi 
(mod 2).! 


e Since we can think of the state of a single qubit register as a vector in two dimensional space, 
a natural operation is for any angle 0, to rotate the single qubit by 9. That is, map |0) to 
cos 0 |0) +sin0 |1), and map |1) to —sin0|0) +cos0|1). Note that rotation by an angle of 7 
(i.e., 180°) is equal to flipping the sign of the vector (i.e., the map v => —v). 


e One simple two qubit operation is exchanging the two bits with one another. That is, mapping 
(01) ++ |10) and |10) + |01), with |00) and |11) being mapped to them selves. Note that by 
combining these operations we can reorder the qubits of an n-bit register in any way we see 
fit. 


e Another two qubit operation is the controlled-NOT operation: it performs a NOT on the first 
bit iff the second bit is equal to 1. That is, it maps |01) > |11) and |11) > |01), with |10) 
and |11) being mapped to themselves. 


‘Note the similarity to the definition of the Walsh-Hadamard code described in Chapter 17, Section 17.5. 
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e The Tofolli operation is the three qubit operation that can be called a “controlled-controlled- 
NOT”: it performs a NOT on the first bit iff both the second and third bits are equal to 1. 
That is, it maps |011) to |111) and vice versa, and maps all other basis states to themselves. 


Exercise 2 asks you to write down explicitly the matrices for these operations. 


We define quantum computation as consisting of a sequence of elementary operations in an 
analogous way to our previous definition of probabilistic computation (Definition 20.6): 


DEFINITION 20.13 (QUANTUM COMPUTATION) 

Let f : {0,1}* — [0,1% and T : N — N be some functions. We say that f is 
computable in quantum T(n)-time if for every n € N and x € {0,1}", f(x) can be 
computed by the following process: 


1. Initialize an m qubit quantum register to the state |z0"~™) (i.e., £ padded 
with zeroes), where m < T(n). 


2. Apply one after the other T(n) elementary quantum operations Fj,..., Fr to 
the register (where we require that there is a polynomial-time TM that on 
input 17,17) outputs the descriptions of F),... , Fr). 


3. Measure the register and let Y denote the obtained value. (That is, if v is the 
final state of the register, then Y is a random variable that takes the value y 
with probability |v,|? for every y € [0,1)”.) 


Denoting £ = |f(x)|, we require that the first £ bits of Y are equal to f(x) with 
probability at least 2/3. 


The following class aims to capture the decision problems with efficient algorithms on quantum 
computers: 


DEFINITION 20.14 (THE CLASS BQP) 
A Boolean function f : {0,1}* — {0,1} is in BQP if there is some polynomial p : N > N such that 
f is computable in quantum p(n)-time. 
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QUANTUM COMPUTATION: SUMMARY OF NOTATIONS. 

The state of an m-qubit register is represented by a unit vector v € C?” 
such that the register takes the value x with probability |v,|?. 

An operation on the register is a function F : C?” — C?” that is unitary. 
(i.e., linear and norm preserving). 

An elementary operation only depends and modifies at most three bits of 
the register. 

A computation of a function f on input x € {0,1}" involves initializing the 
register to the state |z0™~”), applying a sequence of elementary operations 
to it, and then measuring its value. 


REMARK 20.15 

Readers familiar with quantum mechanics or quantum computing may notice that we did not allow 
in our definition of quantum computation several features that are allowed by quantum mechanics. 
These include mized states, that involve both quantum superposition and probability, measuring 
in different basis than the standard basis, and performing partial measurements during the com- 
putation. However, none of these features adds to the computing power of quantum computers. 


20.3.1 Universal quantum operations 


Can we actually implement quantum computation? This is an excellent question, and no one really 
knows. However, one hurdle can be overcome: even though there is an infinite set of possible 
elementary operations, all of them can be generated (or at least sufficiently well approximated) by 
the Hadamard and Tofolli operations described in Example 20.12. In fact, every operation that 
depends on k qubits can be approximated by composing 20(*) of these four operations (times an 
additional small factor depending on the approximation’s quality). Using a counting/dimension 
argument, it can be shown that some unitary transformations do indeed require an exponential 
number of elementary operations to compute (or even approximate). 

One useful consequence of universality is the following: when designing quantum algorithms we 
can assume that we have at our disposal the all operations that depend on k qubits as elementary 
operations, for every constant k (even if k > 3). This is since these can be implemented by 3 qubit 
elementary operations incurring only a constant (depending on k) overhead. 


20.3.2 Spooky coordination and Bell’s state 


To get our first glimpse of how things behave differently in the quantum world, we will now show 
how quantum registers and operations help us win the game described in Section 20.1.2 with higher 
probability than can be achieved according to pre-quantum “classical” physics. 

Recall that the game was the following: 


1. The verifier chooses random x,y Er {0,1} and sends x to Alice and y to Bob, collecting their 
respective answers a and b. 


2. It accepts iff xA y =a@®b. In other words, it accepts if either (x,y) 4 (1,1) and a = b or 
x=y= land a £b. 
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It was shown in Section 20.1.2 that if Alice and Bob can not coordinate their actions then the 
verifier will accept with probability at most 3/4, but quantum effects can allow them to bypass this 
bound as follows: 


1. 


Alice and Bob prepare a 2-qubit quantum register containing the EPR state |00) + |11). 
(They can start with a register initialized to |00) and then apply an elementary operation 
that maps the initial state to the EPR state.) 


. Alice and Bob split the register - Alice takes the first qubit and Bob takes the second qubit. 


(The components containing each qubit of a quantum register do not necessarily need to be 
adjacent to one another.) 


. Alice receives the qubit x from the verifier, and if x = 1 then she applies a rotation by 7/8 


(22.5°) operation to her qubit. (Since the operation involves only her qubit, she can perform 
it even after the register was split.) 


. Bob receives the qubit y from the verifier, and if y = 1 he applies a rotation by by —7/8 


(—22.5°) operation to his qubit. 


. Both of them measure their respective qubits and output the values obtained as their answers 


a and b. 


Note that the order in which Alice and Bob perform their rotations and measurements does not 
matter - it can be shown that all orders yield exactly the same distribution (e.g., see Exercise 3). 
While splitting a quantum register and applying unitary transformations to the different parts 
may sound far fetched, this experiment had been performed several times in practice, verifying the 
following predictions of quantum mechanics: 


‘THEOREM 20.16 
Given the above strategy for Alice and Bob, the verifier will accept with probability at least 0.8. 


PROOF: Recall that Alice and Bob win the game if they output the same answer when (x,y) Æ 
(1,1) and a different answer otherwise. The intuition behind the proof is that in the case that 
(x,y) # (1,1) then the states of the two qubits will be “close” to one another (the angle between 
them is less than 7/8 or 22.5°) and in the other case the states will be “far” (having angle 7/4 or 
45°). Specifically we will show that: 


(1) If x = y = 0 then a = b with probability 1. 


(2) If x Æ y then a = b with probability cos?(7/8) > 0.85 


(3) If x = y = 1 then a = b with probability 1/2. 


Implying that the overall acceptance probability is at least 1 + 50.85 + - = 0.8. 

In the case (1) both Alice and Bob perform no operation on their register, and so when measured 
it will be either in the state |00) or |11), both resulting in Alice and Bob’s outputs being equal. To 
analyze case (2), it suffices to consider the case that z = 0, y = 1 (the other case is symmetrical). 
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In this case Alice applies no transformation to her qubit, and Bob rotates his qubit in a —7/8 angle. 
Imagine that Bob first makes the rotation, then Alice measures her qubit and then Bob measures 
his (this is OK as the order of measurements does not change the outcome). With probability 1/2, 
Alice will get the value 0 and Bob's qubit will collapse to the state |0) rotated by a —7/8 angle, 
meaning that when measuring Bob will obtain the value 0 with probability cos?(7/8). Similarly, if 
Alice gets the value 1 then Bob will also output 1 with cos?(1/8) probability. 

To analyze case (3), we just use direct computation. In this case, after both rotations are 
performed, the register’s state is 


(cos(7/8) |0} + sin(7/8) |1)) (cos(a/8) [0) — sin(7/8) |1)) + 
(— sin(7/8) |0) + cos(7/8) |1)) (sin(7/8) |0) + cos(7/8) |1)) = 
(cos”(7/8) - sin? (7/8)) 100) — 2sin(7/8) cos(7/8) |O1)+ 
2sin(7/8) cos(7/8) |10) + (cos*(7r/8) — sin? (r /8)) 111). 


But since 


cos?(1/8) — sin?(7/8) = cos(7/4) = 2 


TF sin(7/4) = 2sin(7/8) cos(7/8), 


all coefficients in this state have the same absolute value and hence when measured the register 
will yield either one of the four values 00,01,10 and 11 with equal probability 1/4. E 


20.4 Quantum programmer’s toolkit 


Quantum algorithms have some peculiar features that classical algorithm designers are not used to. 
The following observations can serve as a helpful “bag of tricks” for designing quantum algorithms: 


e If we can compute an n-qubit unitary transformation U in T' steps then we can compute the 
transformation Controlled-U in O(T) steps, where Controlled-U maps a vector [17 ... Up Un+1) 
to |U (z1... £n)£n+1) if In+1 = 1 and to itself otherwise. 


The reason is that we can transform every elementary operation F in the computation of U 
to the analogous “Controlled-F” operation. Since the “Controlled-F” operation depends on 
at most 4 qubits, it can be considered also as elementary. 


For every two n-qubit transformations U, U”, we can use this observation twice to compute 
the transformation that invokes U on 21 ...2n if 2n+1 = 1 and invokes U’ otherwise. 


e Every permutation of the standard basis is unitary. That is, any operation that maps a vector 
|x) into |[r(x)) where r is a permutation of {0,1}” is unitary. Of course, this does not mean 
that all such permutations are efficiently computable in quantum polynomial time. 


e For every function f : {0,1}” — (0,1P”, the function x,y > zx, (y O f(zx)) is a permutation 
on {0,1}"'* (in fact, this function is its own inverse). In particular, this means that we 
can use as elementary operations the following “permutation variants” of AND, OR and 
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copying: (1) |r122%3) > |x22(13 ® (41 A2%2))) , (2) [1,1223) > |1,12(123 O (11Vx2))), and 
(3) [1,12) > [11 (xı O x2)). Note that in all these cases we compute the “right” function if 
the last qubit is initially equal to 0. 


elf f : {0,1} => {0,1} is a function computable by a size-T Boolean circuit, then the 
following transformation can be computed by a sequence of O(T) elementary operations: for 
every x € {0,1}”,y € {0,1}",z € {0,1}" 


lx, y, 2) > |z, (y f(x)),07) if z = 07. 


(We don’t care on what the mapping does for z 4 07.) 


The reason is that by transforming every AND, OR or NOT gate into the corresponding 
elementary permutation we can ensure that the i” qubit of z contains the result of the it” 
gate of the circuit when executed on input x. We can then XOR the result of the circuit into 
y using £ elementary operations and run the entire computation backward to return the state 
of z to Of. . 


e We can assume that we are allowed to make a partial measurement in the course of the 
algorithm, and then proceed differently according to its outcome. That is, we can measure a 
some of the qubits of the register. Note that if the register is at state v and we measure its it” 
qubit then with probability >... —1 |v.|? we will get the answer “1” and the register's state 
will change to (the normalized version of) the vector >... —¡ Vz |). Symmetrically, with 
probability >7...,-0 \v.|? we will get the answer “0” and the new state will be 0 Vz |2). 


This is allowed since an algorithm using partial measurement can be replaced with an algo- 
rithm not using it with at most a constant overhead (see Exercise 4). 


e Since the 1-qubit Hadamard operation maps |0} to the uniform state |0) + |1), it can be used 
to simulate tossing a coin: we simply take a qubit in our workspace that is initialized to 0, 
apply Hadamard to it, and measure the result. 


Together, the last three observations imply that quantum computation is at least as powerful 
as “classical” non-quantum computation: 


THEOREM 20.17 
BPP C BQP. 


20.5 Grover’s search algorithm. 


Consider the NP-complete problem SAT of finding, given an n-variable Boolean formula y, whether 
there exists an assignment a € {0,1}” such that y(a) = 1. Using “classical” deterministic or 
probabilistic TM’s, we do not know how to solve this problem better than the trivial poly(n)2”-time 
algorithm.” We now show a beautiful algorithm due to Grover that solves SAT in poly(n)2”/?-time 
on a quantum computer. This is a significant improvement over the classical case, even if it falls 


? There are slightly better algorithms for special cases such as 3SAT. 
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way short of showing that NP C BQP. In fact, Grover’s algorithm solves an even more general 
problem: for every polynomial-time computable function f : {0,1}" — {0,1} (even if f is not 
expressed as a small Boolean formula”), it finds in poly(n)2”/? time a string a such that f(a) = 1 
(if such a string exists). 


Grover’s algorithm is best described geometrically. We assume that the function f has a single 
satisfying assignment a. (The techniques described in Chapter 9, Section 9.3.1 allow us to reduce 
the general problem to this case.) Consider an n-qubit register, and let u denote the uniform state 


vector of this register. That is, u = 2/7 >zeto,1y” |£). The angle between u and |a) is equal to the 
1 

gn/2° 

smaller than 7/2 (90°), and hence we denote it by 7/2 — 0, where sin 0 = 


n is sufficiently large, 0 > JaA (since for small 0, sin 0 ~ 6). 


Since this is a positive number, this angle is 


1 
2n/2 


inverse cosine of their inner product (u, |a)) = 


and hence, assuming 


ja> 


Figure 20.1: Grover’s algorithm finds the string a such that f(a) = 1 as follows. It starts with the uniform vector 
u whose angle with |a} is 7/2 — 0 for 0 ~ 2-"/? and at each step transforms the state of the register into a vector 
that is 20 radians closer to |a). After O(1/0) steps, the state is close enough so that measuring the register yields 
la) with good probability. 


The algorithm starts with the state u, and at each step it gets nearer the state |a) by trans- 
forming its current state to a state whose angle with |a) is smaller by 20 (see Figure 20.1). Thus, 
in O(1/0) = O(2"/2) steps it will get to a state v whose inner product with |a) is larger than, say, 
1/2, implying that a measurement of the register will yield a with probability at least 1/4. 


The main idea is that to rotate a vector w towards the unknown vector |a) by an angle of 0, 
it suffices to take two reflections around the vector u and the vector e = ) y 1a) (the latter is 
the vector orthogonal to |a) on the plane spanned by u and |a)). See Figure 20.2 for a “proof by 
picture”. 

To complete the algorithm’s description, we need to show how we can perform the reflections 
around the vectors u and e. That is, we need to show how we can in polynomial time transform 
a state w of the register into the state that is w’s reflection around u (respectively, e). In fact, 
we will not work with an n-qubit register but with an m-qubit register for m that is polynomial in 
n. However, the extra qubits will only serve as “scratch workspace” and will always contain zero 
except during intermediate computations, and hence can be safely ignored. 


3We may assume that f is given to the algorithm in the form of a polynomial-sized circuit. 
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Step 1: Reflect around e 


ja> 
A 


Step 2: Reflect around u 


la> 


Figure 20.2: We transform a vector w in the plane spanned by |a) and u into a vector w” that is 20 radians close 
to |a) by performing two reflections. First, we reflect around e = >, z, |£) (the vector orthogonal to |a) on this 
plane), and then we reflect around u. If the original angle between w and |a) was 7/2 — 0 — a then the new angle 
will be 7/2 — 0 — a — 20. 


Reflecting around e. Recall that to reflect a vector w around a vector v, we express W as 
av + vt (where v+ is orthogonal to v) and output av — v+. Thus the reflection of w around e is 
equal to > y. ¿q Wx |£) — Wa |a}. Yet, it is easy to perform this transformation: 


1. Since f is computable in polynomial time, we can compute the transformation |xro) => 
|x(o O f(x))) in polynomial (this notation ignores the extra workspace that may be needed, 
but this won't make any difference). This transformation maps |x0) to |10) for x 4 a and 
|a0) to lal). 


2. Then, we apply the elementary transformation that multiplies the vector by —1 if ø = 1, and 
does nothing otherwise. This maps |x0) to [10) for x 4 a and maps lal) to —|a1). 


3. Then, we apply the transformation |zo) > |x(o O f(x))) again, mapping |x0) to |10) for 
x #a and maps |al) to |a0). 


The final result is that the vector |z0) is mapped to itself for x 4 a, but |a0) is mapped to 
—|a0). Ignoring the last qubit, this is exactly a reflection around la). 


Reflecting around u. To reflect around u, we first apply the Hadamard operation to each qubit, 
mapping u to |0). Then, we reflect around |0) (this can be done in the same way as reflecting 
around |a}, just using the function g : {0,1}” — {0,1} that outputs 1 iff its input is all zeroes 
instead of f). Then, we apply the Hadamard operation again, mapping |0) back to u. 

Together these operations allow us to take a vector in the plane spanned by |a) and u and rotate 
it 20 radians closer to |a}. Thus if we start with the vector u, we will only need to repeat them 
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O(1/0) = O(27/?) to obtain a vector that, when measured, yields |a} with constant probability. 
For the sake of completeness, Figure 20.3 contains the full description of Grover’s algorithm. Mi 
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Grover’s Search Algorithm. 


such that f(a) = 1, find a. 


Goal: Given a polynomial-time computable f : {0,1}" — {0,1} with a unique a € {0,1}” 


Quantum register: We use an n + 1 + m-qubit register, where m is large enough so we 
can compute the transformation |1007) > |a(o O f(x))0”). 


Operation 


State (neglecting normalizing factors) 


Apply Hadamard operation to first n qubits. 


For i=1,...,2"/? do: 


Step 1: Reflect around e = ? y. ¿q |T): 

1.1 Compute [z007%) > [x(0 8 f(x))0"™") 

1.2 If o = 1 then multiply vector by —1, otherwise 
do not do anything. 

1.3 Compute |x00"*) > |x(0 @ f(x))0™). 


Step 2: Reflect around u: 
2.1 Apply Hadamard operation to first n qubits. 


2.2 Reflect around |0): 

2.2.1 If first n-qubits are all zero then flip n + 1% 
qubit. 

2.2.2 If n+ 1% qubit is 1 then multiply by —1 

2.2.3 If first n-qubits are all zero then flip n + 1% 
qubit. 


2.3 Apply Hadamard operation to first n qubits. 
Measure register and let a’ be the obtained value in 


the first n qubits. If f(a’) = 1 then output a’. Oth- 
erwise, repeat. 


Initial state: [00 } 

u |0} (where u denotes Szeto} (2) 
vi Jot) 

We let v! = u and maintain the invariant 
that (v’,|a)) = sin(i0), where 9 ~ 27”/? is 
such that (u,|a)) = sin(6) 


wi lor) = Lega va le) 07) 
vi |a)|00™). (w? is vê reflected around 


aa (5)-) 

(wi, u) 107%) (074) + ¿on as |x) [004 ), 
for some coefficients a,’s (given by a, = 
a) 

(w*, u) 0%) [107) + Dogon Ox |x) 07) 


—(w', u) 07) [107) + Y. ¿on Oe |x) |0} 
—(ww*, u) |0") [07H ) +37, ¿on Oe |x) [07+) 


yitl [0 (where vit! is w’ reflected 


around u) 


Figure 20.3: Grover’s Search Algorithm 
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20.6 Simon’s Algorithm 


Although beautiful, Grover’s algorithm still has a significant drawback: it is merely quadratically 
faster than the best known classical algorithm for the same problem. In contrast, in this section we 
show Simon’s algorithm that is a polynomial-time quantum algorithm solving a problem for which 
the best known classical algorithm takes exponential time. 

Simon’s algorithm solves the following problem: given a polynomial-time computable function 
f : {0,1}” — {0,1}” such that there exists a € (0, 1)" satisfying f(x) = f(y) iff x = y@a for every 
x,y € (0,1)”, find this string a. 

Two natural questions are (1) why is this problem interesting? and (2) why do we believe it is 
hard to solve for classical computers? The best answer to (1) is that, as we will see in Section 20.7, a 
generalization of Simon's problem turns out to be crucial in the quantum polynomial-time algorithm 
for famous integer factorization problem. Regarding (2), of course we do not know for certain that 
this problem does not have a classical polynomial-time algorithm (in particular, if P = NP then 
there obviously exists such an algorithm). However, some intuition why it may be hard can be 
gleaned from the following black box model: suppose that you are given access to a black box (or 
oracle) that on input x € (0, 1)”, returns the value f(x). Would you be able to learn a by making 
at most a subexponential number of queries to the black box? It is not hard to see that if a is chosen 
at random from {0,1}" and f is chosen at random subject to the condition that f(x) = f(y) iff 
x = ypa then no algorithm can successfully recover a with reasonable probability using significantly 
less than 2”/2 queries to the black box. Indeed, an algorithm using fewer queries is very likely to 
never get the same answer to two distinct queries, in which case it gets no information about the 
value of a. 


20.6.1 The algorithm 


Simon’s algorithm is actually quite simple. It uses a register of 2n + m qubits, where m is the 
number of workspace bits needed to compute f. (Below we will ignore the last m qubits of the 
register, since they will be always set to all zeroes except in intermediate steps of f’s computation.) 
The algorithm first uses n Hadamard operations to set the first n qubits to the uniform state and 
then apply the operation luz) > |r(z0 f(x)) to the register, resulting (up to normalization) in 
the state 

Y lz)lf@))= Y (7) +e @))|f@)). (1) 


xe{0,1}” xe {0,1}” 


We then measure the second n bits of the register, collapsing its state to 


|v f(x)) + (10 a) f(x) (2) 


for some string x (that is chosen uniformly from {0,1}”). You might think that we're done as the 
state (2) clearly encodes a, however we cannot directly learn a from this state: if we measure the 
first n bits we will get with probability 1/2 the value x and with probability 1/2 the value x@a. Even 
though a can be deduced from these two values combined, each one of them on its own yields no 
information about a. (This point is well worth some contemplation, as it underlies the subtleties 
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involved in quantum computation and demonstrates why a quantum algorithm is not generally 
equivalent to performing exponentially many classical computation in parallel.) 

However, consider now what happens if we perform another n Hadamard operations on the first 
n bits. Since this maps x to the vector Pr al aks ly), the new state of the first n bits will be 


Y (97% + 122") py) = Y (79 + (=1)74(—1)") [y) (3) 
y y 


For every y € [0,1)”, the y!” coefficient in the state (3) is nonzero if and only if if and only if 
a O y =0, and in fact if measured, the state (3) yields a uniform y € {0,1}” satisfying a O y = 0. 

Repeating the entire process k times, we get k uniform strings y1,..., yx satisfying y © a = 0 
or in other words, k linear equations (over the field GF(2)) on the variables a;,...,an. It can be 
easily shown that if, say, k > 2n then with high probability there will be n— 1 linearly independent 
equations among these (see Exercise 5), and hence we will be able to retrieve a from these equations 
using Gaussian elimination. For completeness, a full description of Simon’s algorithm can be found 
in Figure 20.4. 


Simon’s Algorithm. 

Goal: Given a polynomial-time computable f : {0,1}” — {0,1}" such that there is some 
a € {0,1}” satisfying f(x) = f(y) iff y= x @a for every x,y € {0,1}”, find a. 

Quantum register: We use an 2n + m-qubit register, where m is large enough so we 
can compute the transformation |120") > |x(z ® f(1))0”). (Below we ignore the last m 
qubits of the register as they will always contain 0” except in intermediate computations 


of f.) 
Operation State (neglecting normalizing factors) 
Initial state: |0%) 
Apply Hadamard operation to first n qubits. Y y [107 ) 
Compute |xz) > |r(y @ f(x))) die Ef (2)) = DU, (le) + |e @ a)) [f(2)) 
Measure second n bits of register. (jz) + |x @a)) |f(x)> 
Apply Hadamard to first n bits. (E, Ena + (-1)%0Y) \v)) Fa) = 


22 y:aoy=0(—1)° ly) |F(z)) 


Measure first n qubits of register to obtain a value y 
such that y O a = 0. Repeat until we get a sufficient 
number of linearly independent equations on a. 


Figure 20.4: Simon's Algorithm 


20.7 Shor’s algorithm: integer factorization using quantum com- 
puters. 


The integer factorization problem is to find, given an integer N, the set of all prime factors of N 
(i.e., prime numbers that divide N). By a polynomial-time algorithm for this problem we mean an 
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algorithm that runs in time polynomial in the description of N, i.e., poly(log(N)) time. Although 
people have been thinking about the factorization problem in one form or another for at least 2000 
years, we still do not know of a polynomial-time algorithm for it: the best classical algorithm takes 
roughly 2008 N pa steps to factor N [?]. In fact, the presumed difficulty of this problem underlies 
many popular encryption schemes (such as RSA). Therefore, it was quite a surprise when in 1994 
Peter Shor showed a quantum polynomial-time algorithm for this problem. To this day it remains 
the most famous algorithm for quantum computers, and the strongest evidence that BQP may 
contain problems outside of BPP. 


The order-finding problem. Rather than showing an algorithm to factor a given number JN, 
we will show an algorithm for a related problem: given a number A with gcd(A, N) = 1, find the 
order of A modulo N, defined to be the smallest positive integer r such that 4” = 1 (mod N). 
Using elementary number theory, it is fairly straightforward to reduce the task of factoring N to 
solving this problem, and we defer the description of this reduction to Section 20.7.3. 


REMARK 20.18 

It is easy to see that for every positive integer k, if A* = 1 (mod N) then r divides k. (Indeed, 
otherwise if k = er +d for c € Z and d € {1,..,r — 1} then A? = 1 (mod N), contradicting the 
minimality of r.) Similarly, for every x, y it holds that A” = AY (mod N) iff x — y is a multiple of 
r. Therefore, the order finding problem can be defined as follows: given the function f : N > N 
that maps x to A” (mod N) and satisfies that f(x) = f(y) iff r|xz — y, find r. In this notation, the 
similarity to Simon’s problem becomes more apparent. 


20.7.1 Quantum Fourier Transform over Zym. 


The main tool used to solve the order-finding problem is the quantum Fourier transform. We have 
already encountered the Fourier transform in Chapter 19, but will now use a different variant, which 
we call the Fourier transform over Zm where M = 2™ for some integer M. Recall that Zm is the 
group of all number in (0,..., M — 1) with the group operation being addition modulo M. The 
Fourier transform over this group, defined below, is a linear and in fact unitary operation from C?” 
to C?". The quantum Fourier transform is a way to perform this operation by composing O(m?) 
elementary quantum operations (operations that depend on at most three qubits). This means that 
we can transform a quantum system whose register is in state f to a system whose register is in the 
state corresponding to the Fourier transform f of f. This does not mean that we can compute in 
O(m?) the Fourier transform over Zm - indeed this is not sufficient time to even write the output! 
Nonetheless, this transformation still turns out to be very useful, and is crucial to Shor’s factoring 
algorithm in the same way that the Hadamard transformation (which is a Fourier transform over 
the group {0,1}” with the operation $) was crucial to Simon’s algorithm. 


Definition of the Fourier transform over Zym. 


Let M = 2” and let w = e271/M. Note that w™ = 1 and w£ # 1 for every positive integer K < N 
(we call such a number w a primitive M“ root of unity). A function x : Zm => C is called a 
character of Zm if x(y+z) = x(y)x(<) for every y, 2 € Zm. Zm has M characters [xx Prez, where 
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Xa(y) = wr. Let Xz = Xz/VM (this factor is added for normalization), then the set {Xzhnczry is 
an orthonormal basis of the space C™ since 


M-1 


inka) = h So oA = BS ome 


z=0 


which is equal to 1 if x = y and to + IoC Y L ijf # y (the latter equality follows by the 


1—wrt=Yy 
LM 


formula for the sum of a geometric series and the fact that w = 1 for every £). 


DEFINITION 20.19 
For f a vector in CY, the Fourier transform of f is the representation of f in the basis {Xz}. 


We let f (x) denote the coefficient of Y_¿ in this representation. Thus f = ear S F(x )X-2 and so 
fir) =f; x) = ir wY f(x). We let FTu(f) denote the vector (f(0),..., F(M — 1)). 
The function FT y is a unitary operation from C™ to C™ and is called the Fourier transform over 
Vie 


Fast Fourier Transform 


Note that 


== Y) Fw 409A Y Fly)? 


yEZ mM a even yEZ m,y odd 


Now since w? is an M/2th root of unity and wM/2 = _1, letting W be the M/2 diagonal matrix 


with diagonal w®, w™/2-1 we get that 
FTu(f)iow = FT r1/2l feven) + W FTmj2( foda) (4) 
FTu(f) nigh = FT m/2lfeven) — WFTm/2l foda) (5) 


where for an M-dimensional vector v, we denote by Veven (resp. Voda) the M/2-dimensional vector 
obtained by restricting v to the coordinates whose indices have least significant bit equal to 0 (resp. 
1) and by Viow (resp. Vhigh) the restriction of v to coordinates with most significant bit 0 (resp. 
1). 

Equations (4) and (5) are the crux of the well known Fast Fourier Transform (FFT) algorithm 
that computes the Fourier transform in O(M log M) (as opposed to the naive O(M?)) time. We 
will use them for the quantum Fourier transform algorithm, obtaining the following lemma: 


LEMMA 20.20 
There is an O(m?)-step quantum algorithm that transforms a state f = Y pez, f(x) |x) into the 


state f = sta f(z) |x), where f(a) = Ja yc Zn w f(x). 


Quantum Fourier transform: proof of Lemma 20.20 


To prove Lemma 20.20, we use the following algorithm: 
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Quantum Fourier Transform FTy 
Initial state: f => 27, f (2) |x) 
Final state: f = Dez, f(a) |£). 
Operation State (neglecting normalizing factors) 
F= Deez, Ha) la) 

Recursively run FTy/2 on m—1 most | (FTum/2feven) |0) + (FTmj2foaa) |1) 
significant qubits 
If LSB is 1 then compute W on m—1 | (FTmj2feven) |0} + (WFT»r1/2foda) |1) 
most significant qubits (see below). 
Apply Hadmard gate H to least sig- 
nificant qubit. 


(FT 4/2 feven) (|0) a 11) T 
(WWFTyy2foda) 10) — |1)) = 

(FT 4/2 feven + FTm/2foda) |0) + 
(FTmj2feven = W FTmj2foda) 11) 
Move LSB to the most significant po- | |0)(FT11/2feven + FTm/2foda) + 


sition 1) (FTujofeven — WFTm/2foda) = f 


The transformation W on m — 1 qubits can be defined by |z) > w” = whio 2zi (where zx; is 
the i*” qubit of x). It can be easily seen to be the result of applying for every i € {0,..., m — 2} 


the following elementary operation on the it” qubit of the register: |0) > |0) and |1) > w” |1). 


The final state is equal to f by (4) and (5). (We leave verifying this and the running time to 
Exercise 9.) W 


20.7.2 The Order-Finding Algorithm. 


We now present a quantum algorithm that on input a number A < N, finds the order of A 
modulo N (i.e., the smallest r such that A” = 1 (mod N)). We let m = 3log N and M = 2”. 
Our register will consist of m + log(N) qubits. Note that the function x ++ A” (mod N) can 
be computed in polylog(N) time (see Exercise 6) and so we will assume that we can compute 
the map |x) |y) > lx) ly O A” (mod N),) (where _X, denotes the representation of the number 
X € {0,...,N — 1} as a binary string of length log N).* The order-finding algorithm is as follows: 


“To compute this map we may need to extend the register by some additional qubits, but we can ignore them as 
they will always be equal to zero except in intermediate computations. 
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Order finding algorithm. 

Goal: Given numbers N and A < N such that gcd(A, N) = 1, find the smallest r such 
that A” = 1 (mod N). 

Quantum register: We use an m + n-qubit register, where m = 3log N (and hence in 
particular M > N3). Below we treat the first m bits of the register as encoding a number 


VIM/r) 


(mod ha 


in Zm. 

Operation State (including normalizing factors) 

Apply Fourier transform to the first m bits. J X zezu (27) 107) 

Compute the transformation |x) |y) > | rezy (2) 147 (mod N)) 

|z) ly © (A® (mod N))). 

Measure the second register to get a value yo. 1 [ae mis [Zo + £r) |yo) where xo 


is the smallest number such that A”? = yo 


. ee | M/r] Ds (x r)z 
Apply the Fourier transform to the first register. DIETA (Seer, E otér) ES) 


Measure the first register to obtain a number x € Zm. Find the best rational approxi- 


mation a/b (with a,b coprime) for the fraction 77 with denominator b at most 40M (see 


Section 20.A). If A? = A (mod M) then output b. 


In the analysis, it will suffice to show that this algorithm outputs the order r with probability 
at least 1/poly(log(N)) (we can always amplify the algorithm’s success by running it several times 
and taking the smallest output). 


Analysis: the case that r|M 


We start by analyzing the algorithm in the (rather unrealistic) case that M = rc for some integer 
c. In this case we claim that the value x measured will be equal to ele for random cl € 0,...,r. In 
this case, x/M = c'/r. However, with probability at least Q(1/log(r)), the number c’ will be prime 
(and in particular coprime to r). In this case, the denominator of the rational approximation for 
x/M is indeed equal to r. 

Indeed, for every x € Zm, the absolute value of |z )’s coefficient before the measurement is equal 
(up to some normalization factor) to 


c—1 e-1 c—1 
es roce yu =e yu (6) 
£=0 £=0 £=0 

But if x = cc then w” = wM” = 1, and hence the coefficients of all such «’s are equal to the 


th root 


same positive number. On the other hand, if c does not divide x then then since w” is a c 
of unity, a: wf? = 0 by the formula for sums of geometric progressions. Thus, such a number 


x would be measured with zero probability. 


The case that r {M 


In the general case, we will not be able to show that the value x measured satisfies M|xr. However, 
we will show that with Q(1/logr) probability, (1) xr will be “almost divisible” by M in the sense 


) Ivo) 
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Figure 20.5: A complex number z = a+ ib can be thought of as the two-dimensional vector (a,b) of length 
|z| = Va? +b?. The number 3 = e’’ corresponds to a unit vector of angle @ from the x axis.For any such £, if k is 


1-8] _ 2sin(0/2) 
EE 3=m(r9/2]- We use here the fact 


(proved in the boxed figure) that in a unit cycle, the chord corresponding to an angle a is of length 2sin(a/2). 


not too large (say k < 1/0) then by elementary geometric considerations 


that 0 < xr (mod M) < r/10 and (2) [ar/M] is coprime to r. Condition (1) implies that 
lar — cM| < r/10 for c = [ar/M]. Dividing by rM gives | — €| < wọ. Therefore, £ is a 
rational number with denominator at most N that approximates +, to within 1/(10M) < 1/(2N?). 
It is not hard to see that such an approximation is unique (Exercise 7) and hence in this case the 
algorithm will come up with c/r and output the denominator r. 

Thus all that is left is to prove the following two lemmas: 


LEMMA 20.21 
There exist Q(r/logr) values x € Zm such that: 


1. 0< zr (mod M) < r/10 
2. | ar/M | and r are coprime 


LEMMA 20.22 
If x satisfies 0 < xr (mod M) < r/10 then, before the measurement in the final step of the order- 
finding algorithm, the coefficient of |x) is at least A+). 


PROOF OF LEMMA 20.21: We prove the lemma for the case that r is coprime to M, leaving the 
general case as Exercise 10. In this case, the map x => rx (mod M) is a permutation of Zi, and 
we have a set of at least r/(20 log r) x’s such that xr (mod M) is a prime number p between 0 and 
r/10. For every such x, zr + [r/M|M = p which means that [r/M | can not have a nontrivial 
shared factor with r, as otherwise this factor would be shared with p as well. Mi 


PROOF OF LEMMA 20.22: Let x be such that 0 < xr (mod M) < r/10. The absolute value of 
|x)’s coefficient in the state before the measurement is 


[M/r|-1 


1 bra 
VTM/rlVvM e : (7) 


Setting 6 = w”? (note that since M frz, 3 #1) and using the formula for the sum of a geometric 


series, this is at least 
1-g[ M/r1 
1-6 


Vr 
2M 


r sin(0| M/r |/2 
= M GO > (8) 
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where O = oo is the angle such that 3 = e (see Figure 20.5 for a proof by picture of the 


last equality). Under our assumptions | M/r]0 < 1/10 and hence (using the fact that sina ~ a 
vr 1 


for small angles a), the coefficient of x is at least 357 | M/r | > Er a 


20.7.3 Reducing factoring to order finding. 


The reduction of the factoring problem to the order-finding problem follows immediately from the 
following two Lemmas: 


LEMMA 20.23 
For every nonprime N, the probability that a random X in the set Zi, = {X € |N — 1]: gcd(X, N) = 1} 
has an even order r and furthermore, X"/* 4 +1 (mod N) and X 4 —1 (mod N) is at least 1/4. 


LEMMA 20.24 
For every N and Y, if Y? = 1 (mod N) but Y # +1 (mod N) and Y 4 —1 (mod N), then 
gcd(Y —1,N)>1. 


Together, Lemmas 20.23 and 20.24 show that the following algorithm will output a prime factor 
P of N with high probability: (once we have a single prime factor P, we can run the algorithm 
again on N/P) 


1. Choose X at random from |N — 1]. 


2. If gcd(X, N) > 1 then let K = gcd(X, N), otherwise compute the order r of X, and if r is 
even let K = gcd(X”/ — 1, N). 


3. If K € {1, N} then go back to Step 1. If K is a prime then output K and halt. Otherwise, 
use recursion to output a factor of K. 


Note that if T(N) is the running time of the algorithm then it satisfies the equation T(N) < 
T(N/2) + polylog(V) leading to polylog(N) running time. 
PROOF OF LEMMA 20.24: Under our assumptions, N divides Y? — 1 = (Y — 1)(X +1) but does 
not divide neither Y — 1 or Y + 1. But this means that gcd(Y — 1, N) > 1Z since if Y — 1 and N 
were coprime, then since N divides (Y — 1)(Y + 1), it would have to divide X + 1 (Exercise 8). Mi 


PROOF OF LEMMA 20.23: We prove this for the case that N = PQ for two primes P, Q (the proof 
for the general case is similar and is left as Exercise ??). In this case, by the Chinese Reminder 
Theorem, if we map every number X € Z} to the pair (X (mod P), X (mod Q)) then this map 
is one-to-one. Also, the groups Z$ and Z¿ are known to by cyclic which means that there is a 
number g € [P — 1] such that the map j ++ g (mod P) is a permutation of [P — 1] and similarly 
there is a number h € [Q — 1] such that the map k++ h* (mod P) is a permutation of [Q — 1). 
This means that instead of choosing X at random, we can think of choosing two numbers j, k at 
random from [P—1] and [Q—1] respectively and consider the pair (g/ (mod P),h* (mod Q)) which 
is in one-to-one correspondence with the set of X’s in Zi. The order of this pair (or equivalently, 
of X) is the smallest positive integer r such that g = 1 (mod P) and h*” = 1 (mod Q), which 
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means that P — 1|jr and Q — 1|kr. Now suppose that j is odd and k is even (this happens with 
probability 1/4). In this case r is of the form 2r’ where r’ is the smallest number such that P—1|2jr’ 
and Q —1|kr’ (the latter holds since we can divide the two even numbers k and Q —1 by two). But 
this means that g1("/2 4 1 (mod Q) and A*("/2) = 1 (mod Q). In other words, if we let X be the 
number corresponding to (gf (mod P),h* (mod Q)) then X”/? corresponds to a pair of the form 
(a, 1) where a 4 1. However, since +1 (mod N) corresponds to the pair (+1, +1) and —1 (mod N) 
corresponds to the pair (—1 (mod P),—1 (mod Q)) it follows that X"/? 4 +1 (mod N). E 


20.8 BQP and classical complexity classes 


What is the relation between BQP and the classes we already encountered such as P, BPP and 
NP? This is very much an open questions. It not hard to show that quantum computers are at 
least not infinitely powerful compared to classical algorithms: 

THEOREM 20.25 

BQP C PSPACE 


PROOF SKETCH: To simulate a T-step quantum computation on an m bit register, we need to 
come up with a procedure Coeff that for every i € [T] and x € {0,1}”, the zt” coefficient (up to 
some accuracy) of the register’s state in the i” execution. We can compute Coeff on inputs 2, i 
using at most 8 recursive calls to Coeff on inputs 2”,i— 1 (for the at most 8 strings that agree 
with x on the three bits that the F;’s operation reads and modifies). Since we can reuse the space 
used by the recursive operations, if we let S(¿) denote the space needed to compute Coeff(z, i) 
then S(i) < S(i— 1) + O(£) (where £ is the number of bits used to store each coefficient). 

To compute, say, the probability that if measured after the final step the first bit of the register 
is equal to 1, just compute the sum of Coeff(x,T') for every x € {0,1}". Again, by reusing the 
space of each computation this can be done using polynomial space. W 


Theorem 20.25 can be improved to show that BQP C P*P (where #P is the counting version 
of NP described in Chapter 9), but this is currently the best upper bound we know on BQP. 

Does BQP = BPP? The main reason to believe this is false is the polynomial-time algorithm 
for integer factorization. Although this is not as strong as the evidence for, say NP ¢ BPP 
(after all NP contains thousands of well-studied problems that have resisted efficient algorithms), 
the factorization problem is one of the oldest and most well-studied computational problems, and 
the fact that we still know no efficient algorithm for it makes the conjecture that none exists 
appealing. Also note that unlike other famous problems that eventually found an algorithm (e.g., 
linear programming [?] and primality testing [?]), we do not even have a heuristic algorithm that 
is conjectured to work (even without proof) or experimentally works on, say, numbers that are 
product of two random large primes. 

What is the relation between BQP and NP? It seems that quantum computers only offer 
a quadratic speedup (using Grover’s search) on NP-complete problems, and so most researchers 
believe that NP É BPP. On the other hand, there is a problem in BQP (the Recursive Fourier 
Sampling or RFS problem [BV97]) that is not known to be in the polynomial-hierarchy , and so at 
the moment we do not know that BQP = BPP even if we were given a polynomial-time algorithm 
for SAT. 
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Chapter notes and history 


We did not include treatment of many fascinating aspects of quantum information and computation. 
Many of these are covered in the book by Nielsen and Chuang [NC00]. See also Umesh Vazirani's 
excellent lecture notes on the topic (available from his home page). 

One such area is quantum error correction, that tackles the following important issue: how can 
we run a quantum algorithm when at every possible step there is a probability of noise interfering 
with the computation? It turns out that under reasonable noise models, one can prove the following 
threshold theorem: as long as the probability of noise at a single step is lower than some constant 
threshold, one can perform arbitrarily long computations and get the correct answer with high 
probability [?]. 

Quantum computing has a complicated but interesting relation to cryptography. Although 
Shor’s algorithm and its variants break many of the well known public key cryptosystems (those 
based on the hardness of integer factorization and discrete log), the features of quantum mechanics 
can actually be used for cryptographic purposes, a research area called quantum cryptography (see 
(?]). Shor’s algorithm also spurred research on basing public key encryption scheme on other 
computational problems (as far as we know, quantum computers do not make the task of breaking 
most known private key cryptosystems significantly easier). Perhaps the most promising direction 
is basing such schemes on certain problems on integer lattices (see the book [?] and [?]). 

While quantum mechanics has had fantastic success in predicting experiments, some would 
require more from a physical theory. Namely, to tell us what is the “actual reality” of our world. 
Many physicists are understandably uncomfortable with the description of nature as maintaining 
a huge array of possible states, and changing its behavior when it is observed. The popular science 
book [Bru04] contains a good (even if a bit biased) review of physicists’ and philosophers’ attempts 
at providing more palatable descriptions that still manage to predict experiments. 

On a more technical level, while no one doubts that quantum effects exist at microscopic scales, 
scientists questioned why they do not manifest themselves at the macrosopic level (or at least 
not to human consciousness). A Scientific American article by Yam [Yam97] describes various 
explanations that have been advanced over the years. The leading theory is decoherence, which 
tries to use quantum theory to explain the absence of macroscopic quantum effects. Researchers are 
not completely comfortable with this explanation. The issue is undoubtedly important to quantum 
computing, which requires hundreds of thousands of particles to stay in quantum superposition for 
large-ish periods of time. Thus far it is an open question whether this is practically achievable. 
One theoretical idea is to treat decoherence as a form of noise, and to build noise-tolerance into 
the computation —a nontrivial process. For details of this and many other topics, see the books 
by Kitaev, Shen, and Vyalyi [AA02]. 

The original motivation for quantum computing was to construct computers that are able to 
simulate quantum mechanical systems, and this still might be their most important application 
if they are ever built. Feynman [Fey82] was the first to suggest the possibility that quantum 
mechanics might allow Turing Machines more computational power than classical TMs. In 1985 
Deutsch [Deu85] defined a quantum Turing machine, though in retrospect his definition is unsatis- 
factory. Better definitions then appeared in Deutsch-Josza [DJ92], Bernstein-Vazirani [BV97] and 
Yao [Yao93], at which point quantum computation was firmly established as a field. 
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Exercises 


$1 


§2 
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Prove Claim 20.11. 
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For each one of the following operations: Hadamard, NOT, controlled-NOT, rotation by 7/4, 
and Tofolli, write down the 8 x 8 matrix that describes the mapping induced by applying this 
operation on the first qubits of a 3-qubit register. 


Suppose that a two-bit quantum register is in an arbitrary state v. Show that the following 
three experiments will yield the same probability of output: 


(a) Measure the register and output the result. 
(b) First measure the first bit and output it, then measure the second bit and output it. 


(c) First measure the second bit and output it, then measure the first bit and output it. 


Suppose that f is computed in T time by a quantum algorithm that uses a partial measure- 
ments in the middle of the computation, and then proceeds differently according to the result 
of that measurement. Show that f is computable by O(T) elementary operations. 


Prove that if for some a € {0,1}", the strings yi,...,Yn-—1 are chosen uniformly at random 
from {0,1}” subject to y; © a = 0 for every i € [n — 1], then with probability at least 1/10, 
there exists no nonzero string a’ % a such that y; © a’ = 0 for every i € [n — 1]. (In other 
words, the vectors y1,...,Yn-1 are linearly independent.) 


Prove that given 4,1 € {0,..., M — 1}, we can compute A” (mod M) in time polynomial in 
log M. 


‘uotsuedxo Areulq s,1 Suisn Aq Y perou? 10} WIYYLIOSyTe ue moys 
‘UL “Y OWOS 103 yZ = T EY} ƏSVI oy} Sutajos Aq 9.1898 :JUIH 


Prove that for every a < 1, there is at most a single rational number a/b such that b < N 
and |a — a/b| < 1/(2N?). 


Prove that if A,B are numbers such that N and A are coprime but N divides AB, then N 
divides B. 
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89 Prove Lemma 20.20. 


$10 Complete the proof of Lemma ?? for the case that r and M are not coprime. That is, 
prove that also in this case there exist at least Q(r/logr) values x’s such that 0 < ra 
(mod M) < r/2 and [ M/x | and r are coprime. 
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§11 (Uses knowledge of continued fractions) Suppose j,r < N are mutually coprime and unknown 
to us. Show that if we know the first 2log N bits of j/r then we can recover j,r in polynomial 
time. 


20.A Rational approximation of real numbers 


A continued fraction is a number of the following form: 


1 
a2 + at... 


Given a real number a > 0, we can find its representation as an infinite fraction as follows: split 
a into the integer part | a | and fractional part a — | a |, find recursively the representation R of 


1/(a — | a |), and then write 


a=laj+z: 


If we stop after Z steps, we get a rational number that can be represented as a/b with a,b coprime. 
It can be verified that b € (22, 22%]. The following theorem is also known: 


THEOREM 20.26 
[?] If a/b is a rational number obtained by running the continued fraction algorithm on a for a 


finite number of steps then |a — a/b| > |a — c/d for every rational number c/d with denominator 
d < b. 


This means that given any number a and bound N, we can use the continued fraction algorithm 
to find in polylog(V) steps a rational number a/b such that b < 16N and a/b approximates a better 
than any other rational number with denominator at most b. 


Chapter 21 
Logic in complexity theory 


VERY SKETCHY 

As mentioned in the book’s introduction, complexity theory (indeed, all of computer science) 
arose from developments in mathematical logic in the first half of the century. Mathematical logic 
continues to exert an influence today, suggesting terminology and choice of problems (e.g., “boolean 
satisfiability” ) as well as approaches for attacking complexity’s central open questions. This chapter 
is an introduction to the basic concepts. 

Mathematical logic has also influenced many other areas of computer science, such as program- 
ming languages, program verification, and model checking. We will not touch upon them, except 
to note that they supply interesting examples of hard computational problems —ranging from 
NP-complete to EXPSPACE-complete to undecidable. 

The rest of the chapter assumes only a nodding familiarity with logic terminology, which we 
now recount informally; for details see a logic text. 

A logic usually refers to a set of rules about constructing valid sentences. Here are a few logics 
we will encounter. Propositional logic concerns sentences such as (p V q) A (=p V r) where p,q,r 
are boolean variables. Recall that the SAT problem consists of determining the satisfiability of 
such sentences. In first order logic, we allow relation and function symbols as well as quantification 
symbols 3 and VY. For instance, the statement VrS(x) Æ x is a first order sentence in which x is 
quantified universally, S() is a unary relation symbol and 4% is a binary relation. Such logics are 
used in well-known axiomatizations of mathematics, such as Euclidean geometry, Peano Arithmetic 
or Zermelo Frankel set theory. Finally, second order logic allows sentences in which one is allowed 
quantification over structures, i.e., functions and relations. An example of a second order sentence 
is ISVxS(a) 4 x, where S is a unary relation symbol. 

A sentence (or collection of sentences) in a logic has no intrinsic “meaning.” The meaning 
—including truth or falsehood—can be discussed only with reference to a structure, which gives 
a way of interpreting all symbols in the sentence. To give an example, Peano arithmetic consists 
of five sentences (“axioms”) in a logic that consists of symbols like S(x), =, + etc. The standard 
structure of these sentences is the set of positive integers, with S() given the intepretation of 
“successor function,” + given the interpretation of addition, and so on. A structure is said to be a 
model for a sentence or a group of sentences if those sentences are true in that structure. 


Finally, a proof system consists of a set of sentences X called axioms and one or more derivation 
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rules for deriving new sentences from the axioms. We say that sentence o can be proved from X, 
denoted X | ø, if it can be derived from > using a finite number of applications of the derivation 
rules. A proveable sentence is called a theorem. 

Note that a theorem is a result of a mechanical (essentially, algorithmic) process of applying 
derivation rules to the axioms. There is a related notion of whether or not ø is logically implied by 
Y, denoted 2 } ø, which means that every model of * is also a model of ø. In other words, there 
is no “counterexample model” in which the axioms © are true but ø is not. The two notions are in 
general different but Godel in his completeness theorem for first order theories exhibited a natural 
set of derivation rules such that logically implied sentences are exactly the set of theorems. (This 
result was a stepping stone to his even more famous incompleteness theorem.) 

Later in this chapter we give a complexity-theoretic definition of a proof system, and introduce 
the area of proof complexity that studies the size of the smallest proof of a mathematical statement 
in a given proof system. 


21.1 Logical definitions of complexity classes 


Just as Church and others defined computation using logic without referring to any kind of com- 
puting machine, it is possible to give “machineless” characterizations of many complexity classes 
using logic. We describe a few examples below. 


21.1.1 Fagin’s definition of NP 


In 1974, just as the theory of NP-completeness was coming into its own, Fagin showed how to 
define NP using second-order logic. We describe his idea using an example. 


EXAMPLE 21.1 
(Representing 3-COLOR) We show how to represent the set of 3-colorable graphs using second 
order logic. 

Let E be a symbol for a binary relation, and Co, C1, C2 be symbols for unary relations, and 
(E, Co, C1, C2) be a first order formula that is a conjunction of the following formulae where 
1+1,1+2 are meant to be understood modulo 3: 


Vu,u E(u,v) = E(v,u) 

Vu Ag=1,2,3 (Cilu) > =(C;41(u) V Ci42(u)) 
VuC;(u) V Cy41(u) V Cy42(u) 

Vu,v E(u,v) > Aia1,2,3(Ci(u) > AC;(v)) 


E 


A SE E EN 


( 
( 
( 
( 


What set of E’s defined on a finite set satisfy 4C94C14C2¢(E, Co, C1, C2)? If E is defined on a 
universe of size n (i.e., u,v take values in this universe) then (1) says that E is symmetric, i.e., it 
may be viewed as the edge set of an undirected graph on n vertices. Conditions (2) and (3) say that 
Co, C1, C2 partition the vertices into three classes. Finally, condition (4) says that the partition is 
a valid coloring. 
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Now we can sketch the general result. To represent a general NP problem, there is a unary 
relation symbol that represents the input (in the above case, E). The witness is a tableau (see 
Chapter 2) of an accepting computation. If the tableau has size n*, the witness can be represented 
by a k-ary relation (in the above case the witness is a 3-coloring, which has representation size 3n 
and hence was represented using 3 unary relations). The first order formula uses the Cook-Levin 
observation that the tableau is correct iff it is correct in all 2 x 3 “windows”. 

The formal statement of Fagin’s theorem is as follows; the proof is left as an exercise. 


THEOREM 21.2 (FAGIN) 
To be written. 


21.1.2 MAX-SNP 
21.2 Proof complexity as an approach to NP versus coNP 


Proof complexity tries to study the size of the smallest proof of a statement in a given proof 
system. First, we need a formal definition of what a proof system is. The following definition due 
to Cook and Reckow focuses attention on the intuitive property that a mathematical proof is “easy 
to check.” 


DEFINITION 21.3 
A proof system consists of a polynomial-time Turing machine M. A statement T is said to be a 
theorem of this proof system iff there is a string m € {0,1}* such that M accepts (T, 7). 

If T is a theorem of proof system M, then the proof complexity of T with respect to M is the 
minimun k such that there is some 7 € {0,1}* for which M accepts (T, 7). 


Note that the definition of theoremhood ignores the issue of the length of the proof, and insists 
only that the M’s running time is polynomial in the input length |T| + |r|. The following is an 
easy consequence of the definition and the motivation for much of the field of proof complexity. 


THEOREM 21.4 
A proof system M in which SAT has polynomial proof complexity exists iff NP = coNP. 


Many branches of mathematics, including logic, algebra, geometry, etc. give rise to proof sys- 
tems. Algorithms for SAT and automated theorem provers (popular in some areas of computer 
science) also may be viewed as proof systems. 


21.2.1 Resolution 


This concerns 
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21.2.3 Polynomial calculus 


21.3 Is P + NP unproveable? 


Chapter 22 


Why are circuit lowerbounds so 
difficult? 


Why have we not been able to prove strong lower bounds for circuits? In 1994 Razborov and Rudich 
formalized the notion of a “natural mathematical proof,” for a circuit lowerbound. They pointed out 
that current lowerbound arguments involve “natural” mathematical proofs, and show that obtaining 
strong lowerbound with such techniques would violate a widely believed cryptographic assumption 
(namely, that factoring integers requires time 2” for some fixed e > 0). Thus presumably we 
need to develop mathematical arguments that are not natural. This result may be viewed as a 
modern analogue of the Baker, Gill, Solovay result from the 1970s (see Chapter ??) that showed 
that diagonalization alone cannot resolve P versus NP and other questions. 

Basically, a natural technique is one that proves a lowerbound for a random function and is 
“constructive.” We formalize “constructive” later but first consider why lowerbound proofs may 
need to work for random functions. 


22.1 Formal Complexity Measures 


Let us imagine at a high level how one might approach the project of proving circuit lower bounds. 
For concreteness, focus on formulas, which are boolean circuits where gates have indegree 2 and 
outdegree 1. It is tempting to use some kind of induction. Suppose we have a function like the one 
in Figure 22.1 that we believe to be “complicated.” Since the function computed at the output is 
“complicated”, intuition says that at least one of the functions on the incoming edges to the output 
gate should also be “pretty complicated” (after all those two functions can be combined with a 
single gate to produce a “complicated” function). Now we try to formalize this intuition, and point 
out why one ends up proving a lowerbound on the formula complexity of random functions. 

The most obvious way to formalize a “complicatedness” is as a function u that maps every 
boolean function on {0,1}" to a nonnegative integer. (The input to y is the truth table of the 
function.) We say that u is a formal complexity measure if it satisfies the following properties: 
First, the measure is low for trivial functions: u(x) < 1 and u(z;) < 1 for all i. Second, we require 
that 
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Figure 22.1: A formula for a hard function. 


e u(f Ag) < wf) + u(g) for all f, 9; and 
e u(fV g) € uf) + u(g) for all f, y. 


For instance, the following function p is trivially a formal complexity measure 


p(f) = 1 + the smallest formula size for f. (1) 


In fact, it is easy to prove the following by induction. 


THEOREM 22.1 
If uu is any formal complexity measure, then u(f) is a lowerbound on the formula complexity of f. 


Thus to formalize the inductive approach outlined earlier, it suffices to define a measure u such 
that (CLIQUE) is high (say superpolynomial). For example, one could try “fraction of inputs for 
which the function agrees with the CLIQUE function” or some suitably modified version of this. In 
general, one imagines that defining a measure that lets us prove a good lowerbound for CLIQUE 
would involve some deep observation about the CLIQUE function. The next lemma seems to show, 
however, that even though all we care about is the CLIQUE function, our lowerbound necessarily 
must reason about random functions. 

LEMMA 22.2 

Suppose u is a formal complexity measure and there exists a function f : {0,1}" — {0,1} such 
that u(f) > c for some large number c. Then for at least 1/4 of all functions g : (0,1)” — {0,1} 
we must have (g) > c/4. 


PROOF: Let g : (0,1)” — {0,1} be any function. Write f as f = h@®g where h = f O g. So 
f=(RAg)V(hAgJ) and Wf) < ulg) + HG) + ulh) + H(A). 

Now suppose for contradiction’s sake that {g : (g) < c/4} contains more than 3/4 of all boolean 
functions on n-bit inputs. If we pick the above function g randomly, then 7,h,h are also random 
(though not independent). Using the trivial union bound we have Pr[All of h,h,g,g have y < 
c/4] > 0. Hence u(f) < c, which contradicts the assumption. Thus the lemma is proved. W 


In fact, the following stronger theorem holds: 


THEOREM 22.3 
If u(f) > c then for all e > 0 and for at least 1 — e of all functions g we have that, 


lao) 


The idea behind the proof of the theorem is to write f as the boolean combination of a small 
number of functions and then proceed similarly as in the proof of the lemma. 
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22.2 Natural Properties 


Moving the above discussion forward, we think of a lowerbound proof as identifying some property 
of “hard” functions that is not shared by “easy” functions. 


DEFINITION 22.4 
A property ® is a map from boolean functions to {0,1}. A P-natural property useful against P/poly 
is a property ® such that: 


1. P(f) = 1 for at least a 1/2” fraction of all boolean functions on n bits (recall that there are 
2?" functions on n bits); 


2. ®(f) = 1 implies that f Z P/poly (or more concretely, that f has circuit complexity at least 
nls”, say); and 


3. Dis computable on n-bit functions in 290) time (i.e., polynomial in the length of the function’s 
truth table). 


The term P-natural refers to requirement (3). The property is useful against P/poly because of 
requirement (2). (Note that this requirement also ensures that ® is not trivial, since it must be 0 for 
functions in P/poly.) Requirement (1) corresponds to our above intuition that circuit lowerbounds 
should prove the hardness of a random function. 

By suitably modifying (2) and (3) we can analogously define, for any complexity class Cı and 
circuit class C2, a C¡-natural property that is useful against circuit class C2. We emphasize that 
when the property is computed, the input is the truth table of a function, whose size is 2”. Thus 
a P-natural property is computed in time 2%” for some constant c > 1 and a PSPACE-natural 
property is computed in space 2”, 


EXAMPLE 22.5 
The result that PARITY is not computable in AC? (Section ??) involved the following steps. (a) 
Show that every AC? circuit can be simplified by restricting at most n—n* input bits so that it then 
becomes a constant function. (b) Show that the PARITY function does not have this property. 
Thus the natural property lurking in this proof is the following: ®(f) = 1 iff for every way of 
assigning values to at most n — n* input bits the function does not become a constant function. 
Clearly, if ®(f) = 1 then f ¢ AC°, so f is useful against AC°. Furthermore, ® can be computed in 
20(1) time — just enumerate all possible choices for the subsets of variables and all ways of setting 
them to 0/1. This running time is polynomial in the length of the truth-table, so ® is P-natural. 
Finally, requirement (1) is also met since almost all boolean functions satisfy 6(f) = 1 (easy to 
check using a simple probability calculation; left as exercise). 
Thinking further, we see that ® is a AC°-natural property that is useful against AC®. 
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EXAMPLE 22.6 

The lowerbound for ACC? circuits described in Section ?? is not natural per se. Razborov and 
Rudich show how to naturalize the proof, in other words change it —while retaining its essence—so 
that it does use a natural property. Recall that every boolean function on n bits can be represented 
by a multilinear polynomial over GF'(3). The space of all n-variate multilinear polynomials forms 
a vector space, whose dimension is N = 2”. Then all multilinear polynomials in n variables of total 
degree less than n/2 form a subspace of dimension N/2 (this assumes n is even), and we denote this 
space by L. For a boolean function f let f be a multilinear polynomial over GF (3) that represents 
f. Then define 9(F) = 1 iff the dimension of the space 


{fu +h: heL} 


is at least 3N/4. It can be checked that ® is 1 for the parity function, as well as for most ran- 
dom functions. Furthermore, rank computations can be done in NC? so it is NC?-natural. The 
technique of Section ?? can be used to show that if ®(f) = 1 then f g ACC'[3); thus © is useful 
against ACC'[3]. 


EXAMPLE 22.7 

The lowerbound for monotone circuits in Section ?? does use constructive methods, but it is 
challenging to show that it applies to a random function since a random function is not monotone. 
Nobody has formulated a good definition of a random monotone function. 


In the definition of natural proofs, requirement (3) is the most controversial in that there is no 
inherent reason why mathematical proofs should go hand in hand with efficient algorithms. 


REMARK 22.8 

“Constructive mathematics” was a movement within mathematics that rejected any proofs of exis- 
tence that did not yield an algorithm for constructing the object. Today this viewpoint is considered 
quaint; nonconstructive proofs are integral to mathematics. 

In our context, “constructive” has a stricter meaning, namely the proof has to yield a polynomial- 
time algorithm. Many proofs that would be “constructive” for a mathematician would be noncon- 
structive under our definition. Surprisingly, even with this stricter definition, proofs in combinato- 
rial mathematics are usually constructive, and —as Razborov and Rudich are pointing out —the 
same is true of current circuit lowerbounds as well. 

In a few cases, combinatorial results initially proved “nonconstructively” later turned out to 
have constructive proofs: a famous example is the Lovasz Local Lemma (discovered in 1974; al- 
gorithmic version is in Beck [Bec91]). The same is true for several circuit lowerbounds—cf. the 
“naturalized” version of the Razborov-Smolensky lowerbound for ACC°[q] mentioned earlier, and 
Raz’s proof [Raz00] of the Babai-Nisan-Szegedy [BNS] lowerbound on multiparty communication 
complexity. 
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22.3 Limitations of Natural Proofs 


The following theorem by Razborov and Rudich explains why we have not been able to use the 
same techniques to obtain an upper bound on P/poly: constructing a P-natural property useful 
against P/poly violates widely believed cryptographic assumptions. 


THEOREM 22.9 (RAZBOROV, RUDICH [RR97]) 

Suppose a P-natural property ® exists that is useful against P/poly. Then there are no strong 
pseudorandom function generators. In particular, FACTORING and DISCRETE LOG can be 
solved in less than 2” time for all e > 0. 


Pseudorandom function generators were defined in Section ??. The definition used a distin- 
guisher polynomial-time machine that is given oracle access to either a truly random function or a 
function from the pseudorandom family. The family is termed pseudorandom if the distinguisher 
cannot distinguish between the two oracles. Now we tailor that more general definition for our nar- 
row purposes in this section. We allow the distinguisher 20(1) time and even allow it to examine the 
truth table of the function! This is without loss of generality since in 20(") time the distinguisher 
could construct the truth table using 2” queries to the oracle. 


DEFINITION 22.10 

A pseudorandom function generator is a function f(k, x) computable in polynomial time where the 
input x has n bits and the “key” k has n° bits, where c > 2 is a fixed constant. Denoting by Fn 
the function obtained by uniformly selecting k € {0,1}" and setting Fn to f(k,-), we have the 
property that the function ensemble F = {Fn}; is “pseudorandom,” namely, for each Turing 
machine M running in time 20), and for all sufficiently large n, 


| Pr[/M (Fn) = 1] — Pr[M(H,,) = 1]| < ae 
where H,, is a random function on {0,1}". 
We will denote f(k,-) by fr. 


Intuitively, the above definition says that if f is a pseudorandom function generator, then for 
a random k, the probability is high that f “looks like a random function” to all Turing machines 
running in time 20‘), Note that fg cannot look random to machines that run in 200°) time since 
they can just guess the key k. Thus restricting the running time to 200) (or to some other fixed 
exponential function such as 20(n*)) is crucial. 

Recall that Section ?? described the Goldreich-Goldwasser-Micali construction of pseudorandom 
function generators f(k,x) using a pseudorandom generator g that stretches n° random bits to 2n° 
pseudorandom (also see Figure 22.2): Let go(k) and gi(k) denote, respectively, the first and last 
n° bits of g(k). Then the following function is a pseudorandom function generator, where MSB(x) 
refers to the first bit of a string zx: 


f(k, £) = MSB(grn © Ye; 19090 © ge (k)). 


The exercises in Chapter 10 explored the security of this construction as a function of the 
security parameter of g; basically, the two are essentially the same. By the Goldreich-Levin theorem 
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Figure 22.2: Constructing a pseudorandom function generator from a pseudorandom generator. 


of Section ??, a pseudorandom generator with such a high security parameter exists if a oneway 
permutation exists and some e > 0, such that every 2” time algorithm has inversion probability 
less than 2". The DISCRETE LOG function —a permutation— is conjectured to satisfy this 
property. As mentioned in Chapter 10, researchers believe that there is a small e > 0 such that the 
worst-case complexity of DISCRETE LOG is 2”, which by random self-reducibility also implies the 
hardness of the average case. (One can also obtain pseudorandom generators using FACTORING, 
versions of which are also believed to be just as hard as DISCRETE LOG.) If this belief is correct, 
then pseudorandom function generators exist as outlined above. (Exercise.) 
Now we can prove the above theorem. 


THEOREM 22.9: Suppose the property ® exists, and f is a pseudorandom function generator. We 
show that a Turing machine can use ® to distinguish fp from a random function. First note that 
fk € P/poly for every k (just hardwire k into the circuit for fx) so the contrapositive of property 
(2) implies that P(f,) = 0. In addition, property (1) implies that Pry, [®(H,,) = 1] > 1/2”. Hence, 


Pr/®(H,,)|;-— P 
lad ) we {0,1}"° 


[®( fx] > 1/2”, 


and thus ® is a distinguisher against f. W 


22.4 My personal view 


Discouraged by the Razborov-Rudich result, researchers (myself included) hardly ever work on 
circuit lowerbounds. Lately, I have begun to think this reaction was extreme. I still agree that 
a circuit lowerbound for say CLIQUE, if and when we prove it, will very likely apply to random 
functions as well. Thus the way to get around the Razborov-Rudich observation is to define 
properties that are not P-natural; in other words, are nonconstructive. I feel that this need not be 
such an insurmountable barrier since a host of mathematical results are nonconstructive. 
Concretely, consider the question of separating NEXP from ACC®, one of the (admittedly 
not very ambitious) frontiers of circuit complexity outlined in Chapter 13. As observed there, 
NEXP + ACC? will follow if we can improve the Babai-Nisan-Szegedy lowerbound of 2(n/2*) 
for k-party communication complexity to Q(n/poly(k)) for some function in NEXP. One line of 
attack is to lowerbound the discrepancy of all large cylinder intersections in the truth table, as we 
saw in Raz's proof of the BNS lowerbound!. (In other words, the “unnatural” property we are 
defining is ® where ®(f) = 1 iff f has high discrepancy and thus high multiparty communication 
complexity.) For a long time, I found this question intimidating because the problem of computing 
the discrepancy given the truth table of the function is coNP-hard (even for k = 2). This seemed 


‘Interestingly, Raz discovered this naturalization of the BNS proof after being briefly hopeful that the original 
BNS proof—which is not natural— may allow a way around the Razborov-Rudich result. 
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to suggest that a proof that the discrepancy is high for an explicit function (which presumably will 
also show that it is high for random functions) must have a nonconstructive nature, and hence will 
be very difficult. Lately, I have begun to suspect this intuition. 

A relevant example is Lovasz’s lowerbound of the chromatic number of the Kneser graph [Lov78]. 
Lowerbounding the chromatic number is coNP-complete in general. Lovasz gives a topological 
proof (using the famous Borsuk-Ulam fixed point theorem) that determines the chromatic number of 
the Kneser graph exactly. From his proof one can indeed obtain an algorithm for solving chromatic 
number on all graphs({MZ02]) —but it runs in PSPACE for general graphs! So if this were a circuit 
lowerbound we could call it PSPACE-natural, and thus “nonconstructive.” Nevertheless, Lovasz’s 
reasoning for the particular case of the Kneser graph is not overly complicated because the graph is 
highly symmetrical. This suggests we should not blindly trust the intuition that “nonconstructive 
= difficult.” 

I fervently hope that the next generation of researchers will view the Razborov-Rudich theorem 
as a guide rather than as a big obstacle! 


Exercises 
$1 Prove Theorem 22.3. 


§2 Prove that a random function satisfies ®(f) = 1 with high probability, where 9 is the property 
defined in Example 22.5. 


$3 Show that if the hardness assumption for discrete log is true, then pseudorandom function 
generators as defined in this chapter exist. 


$4 Prove Wigderson’s observation: P-natural properties cannot prove that DISCRETE LOG 
requires circuits of 2” size. 
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85 (Razborov [Raz92]) A submodular complexity measure is a complexity measure that satisfies 


uf Vg) +u(f Ag) < u(f) + ulg) for all functions f,g. Show that for every n-bit function 
fn, such a measure satisfies u( fn) = O(n). 
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Chapter notes and history 


The observation that circuit lowerbounds may unwittingly end up reasoning about random functions 
first appears in Razborov [Raz89]'s result about the limitations of the method of approximation. 
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We did not cover the full spectrum of ideas in the Razborov-Rudich paper [RR97], where it is 
observed that candidate pseudorandom function generators exist even in the class TC%, which lies 
between ACC? and NC!. Thus natural proofs will probably not allow us to separate even TC? 
from P. 

Razborov’s observation about submodular measures in Problem 5 is important because many 
existing approaches for formula complexity use submodular measures; thus they will fail to even 
prove superlinear lowerbounds. 

In contrast with my limited optimism, Razborov himself expresses (in the introduction to [Raz03]) 
a view that the obstacle posed by the natural proofs observation is very serious. He observes that 
existing lowerbound approaches use weak theories of arithmetic such as Bounded Arithmetic. He 
conjectures that any circuit lowerbound attempt in such a logical system must be natural (and 
hence unlikely to work). But as I mentioned, several theorems even in discrete mathematics use 
reasoning (e.g., fixed point theorems like Borsuk-Ulam) that does not seem to be formalizable in 
Bounded Arithmetic. Thus is my reason for optimism. 

However, somen other researchers are far more pessimistic: they fear that P versus NP may 
be independent of mathematics (say, of Zermelo-Fraenkel set theory). Razborov says that he has 
no intuition about this. 


Appendices 
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DRAFT 


Appendix A 


Mathematical Background. 


This appendix reviews the mathematical notions used in this book. However, most of these are 
only used in few places, and so the reader might want to only quickly review Sections A.1, A.2 and 
A.3, and come back to the other sections as needed. In particular, apart from probability, the first 
part of the book essentially requires only comfort with mathematical proofs and some very basic 
notions of discrete math. 

The topics described in this appendix are covered in greater depth in many texts and online 
sources. Almost all of the mathematical background needed is covered in a good undergraduate 
“discrete math for computer science” course as currently taught at many computer science depart- 
ments. Some good sources for this material are the lecture notes by Papadimitriou and Vazirani 
[PV06], Lehman and Leighton [LL06] and the book of Rosen [Ros06]. 

Although knowledge of algorithms is not strictly necessary for this book, it would be quite 
useful. It would be helpful to review either one of the two excellent recent books by Dasgupta 
et al [DPV06] and Kleinberg and Tardos [K'T06] or the earlier text by Cormen et al [CLRSO1]. 
This book does not require prior knowledge of computability and automata theory, but some basic 
familiarity with that theory could be useful: see Sipser’s book [SIP96] for an excellent introduction. 
Mitzenmacher and Upfal [MU05] and Prabhakar and Raghavan [?] cover both algorithmic reasoning 
and probability. For more insight on discrete probability, see the book by Alon and Spencer [AS00]. 


A.1 Mathematical Proofs 


Perhaps the mathematical prerequisite needed for this book is a certain level of comfort with 
mathematical proofs. While in everyday life we might use “proof” to describe a fairly convincing 
argument, in mathematics a proof is an argument that is convincing beyond any shadow of a doubt.! 
For example, consider the following mathematical statement: 


Every even number greater than 2 is equal to the sum of two primes. 


‘In a famous joke, as a mathematician and an engineer drive in Scotland they see a white sheep on their left side. 
The engineer says “you see: all the sheep in Scotland are white”. The mathematician replies “All I see is that there 
exists a sheep in Scotland whose right side is white”. 


pA.1 (447) 
Complexity Theory: A Modern Approach. ©) 2006 Sanjeev Arora and Boaz Barak. References and attributions are 
still incomplete. 


pA.2 (448) A.1. MATHEMATICAL PROOFS 


This statement, known as “Goldbach’s Conjecture”, was conjectured to be true by Christian 
Goldbach in 1742. In the more than 250 years that have passed since, no one has ever found a 
counterexample to this statement. In fact, it has been verified to be true for all even numbers from 
4 till 100,000, 000, 000, 000, 000. Yet still it is not considered proven, since we have not ruled out 
the possibility that there is some (very large) even number that cannot be expressed as the sum of 
two primes. 

The fact that a mathematical proof has to be absolutely convincing does not mean that it has to 
be overly formal and tedious. It just has to be clearly written, and contain no logical gaps. When 
you write proofs try to be clear and concise, rather than using too much formal notation. When 
you read proofs, try to ask yourself at every statement “am I really convinced that this statement 
is true?”. 

Of course, to be absolutely convinced that some statement is true, we need to be certain 
of what that statement means. This why there is a special emphasis in mathematics on very 
precise definitions. Whenever you read a definition, try to make sure you completely understand 
it, perhaps by working through some simple examples. Oftentimes, understanding the meaning of 
a mathematical statement is more than half the work to prove that it is true. 


EXAMPLE A.1 

Here is an example for a classical mathematical proof, written by Euclid around 300 B.C. Recall 
that a prime number is an integer p > 1 whose only divisors are p and 1, and that every number n 
is a product of prime numbers. Euclid’s Theorem is the following: 


‘THEOREM A.2 
There exist infinitely many primes. 


Before proving it, let’s see that we understand what this statement means. It simply means 
that for every natural number k, there are more than k primes, and hence the number of primes is 
not finite. 

At first, one might think it’s obvious that there are infinitely many primes because there are 
infinitely many natural numbers, and each natural number is a product of primes. However, this is 
faulty reasoning: for example, the set of numbers of the form 3” is infinite, even though their only 
factor is the single prime 3. 

To prove Theorem A.2, we use the technique of proof by contradiction. That is, we assume it is 
false and try to derive a contradiction from that assumption. Indeed, assume that all the primes 
can be enumerated as p1, p2,...,px for some number k. Define the number n = p,po---p, +1. 
Since we assume that the numbers p1,..., pz are all the primes, all of n’s prime factors must come 
from this set, and in particular there is some ¿ between 1 and k such that p; divides n. That is, 
n = pim for some number m. Thus, 


pim = pip2**: pet l 


or equivalently, 


pim — pPip2***Pr=l. 
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But dividing both sides of this equation by p;, we will get a whole number on the left hand side (as 
pi is a factor of pyp2--- pp) and the fraction 1/p; on the right hand side, deriving a contradiction. 
This allows us to rightfully place the QED symbol M and consider Theorem A.2 as proven. 


A.2 Sets, Functions, Pairs, Strings, Graphs, Logic. 


A set contains a finite or infinite number of elements, without repetition or respect to order, for ex- 
ample {2,17,5}, N = {1,2,3,...} (the set of natural numbers), [n] = {1,2,...,n} (the set of natural 
numbers from 1 ro n), R (the set of real numbers). For a finite set A, we denote by |A| the number 
of elements in A. Some operations on sets are: (1) union: AU B = {x:x € Aor x € B}, (2) in- 
tersection: AN B = {x : x € A and z € B}, and (3) substraction: A\ B = {x : x € A and z ¢ B}. 

We say that f is a function from a set A to B, denoted by f : A — B, if it maps any element 
of A into an element of B. If B and A are finite, then the number of possible functions from A to 
B is |B|4|. We say that f is one to one if for every x, w € A with « 4 w, f(x) + f(w). If A,B are 
finite, the existence of such a function implies that |A| < |B|. We say that f is onto if for every 
y € B there exists x € A such that f(x) = y. If A, B are finite, the existence of such a function 
implies that |A| > |B|. We say that f is a permutation if it is both one-to-one and onto. For finite 
A, B, the existence of a permutation from A to B implies that |A| = |B|. 

If A,B are sets, then the A x B denotes the set of all ordered pairs (a,b) with a € A,b € B. 
Note that if A, B are finite then |A x B| = |A| - |B]. We can define similarly A x B x C to be 
the set of ordered triples (a,b,c) with a € A,b€ B,c € C. For n € N, we denote by A” the set 
Ax Ax---x A(n times). We will often use the set {0,1}”, consisting of all length-n sequences 
of bits (i.e., length n strings), and the set {0,1}* = Unso {0,1}” ({0,1}° has a single element: a 
binary string of length zero, which we call the empty word and denote by e). 

A graph G consists of a set V of vertices (which we often assume is equal to the set [n] = 
{1,...,n} for some n € N) and a set E of edges, which consists of unordered pairs (i.e., size two 
subsets) of elements in V. We denote the edge {u,v} of the graph by uv. For v € V, the neighbors 
of v are all the vertices u € V such that uv € E. In a directed graph, the edges consist of ordered 
pairs of vertices, to stress this we sometimes denote the edge (u,v) in a directed graph by uv. One 
can represent an n-vertex graph G by its adjacency matrix which is an n x n matrix A such that 
A; j is equal to 1 if the edge 77 is present in G it and is equal to 0 otherwise. One can think of an 
undirected graph as a directed graph G that satisfies that for every u,v, G contains the edge uv if 
and only if it contains the edge vù. Hence, one can represent an undirected graph by an adjecancy 
matrix that is symmetric (A; j = Aj; for every i,j € [n]). 

A Boolean variable is a variable that can be either TRUE or FALSE (we sometimes identify TRUE 
with 1 and FALSE with 0). We can combine variables via the logical operations AND (A), OR (V) 
and NOT (~, sometimes also denoted by an overline), to obtain Boolean formulae. For example, 
the following is a Boolean formulae on the variables u1, u2, uz: (u1 AU2) V=(uzgAu1). The definitions 
of the operations are the usual: a Ab = TRUE if a = TRUE and b = TRUE and is equal to FALSE 
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otherwise; a = sa = TRUE if a = FALSE and is equal to FALSE otherwise; a V b = =(a V b). If p 
is a formulae in n variables u1,..., Un, then for any assignment of values u € {FALSE, TRUE)” (or 
equivalently, (0,1)”), we denote by y(u) the value of y when its variables are assigned the values 
in u. We say that y is satisfiable if there exists a u such that y(u) = TRUE. 

We will often use the quantifiers V (for all) and 3 (exists). That is, if y is a condition that can 
be TRUE or FALSE depending on the value of a variable x, then we write V,y(x) to denote the 
statement that p is TRUE for every possible value that can be assigned to x. If A is a set then we 
write Vreay(x) to denote the statement that y is TRUE for every assignment for x from the set A. 
The quantifier 4 is defined similarly. Formally, we say that 4,y(x) holds if and only if =(V»=p(1)) 
holds. 


A.3 Probability theory 


A finite probability space is a finite set Q = {w1,...,wy} along with a set of numbers p1,...,pn € 
[0,1] such that pa pi = 1. A random element is selected from this space by choosing w; with 
probability p;. If x is chosen from the sample space Q then we denote this by x Er Q. If no 
distribution is specified then we use the uniform distribution over the elements of Q (i.e., pi = x 
for every 1). 

An event over the space 2 is a subset A C Q and the probability that A occurs, denoted by 
Pr[ 4], is equal to ide APi- To give an example, the probability space could be that of all 2” 
possible outcomes of n tosses of a fair coin (i.e., Q = {0,1}" and p; = 27” for every i € [2”]) and 
the event A can be that the number of coins that come up “heads” (or, equivalently, 1) is even. In 
this case, Pr[A] = 1/2 (exercise). The following simple bound —called the union bound—is often 


used in the book. For every set of events Aj, 49,..., An, 
Pr[U A] < Y PriAi. (1 
i=1 


Inclusion exclusion principle. The union bound is a special case of a more general principle. 
Indeed, note that if the sets A,,...,A, are not disjoint then the probability of U;A; could be 
smaller than > 7, Pr[4;] since we are overcounting elements that appear in more than one set. We 
can correct this by substracting >; <j Pr[A; N Aj] but then we might be undercounting, since we 
subtracted elements that appear in at least 3 sets too many times. Continuing this process we get 


CLAIM A.3 (INCLUSION-EXCLUSION PRINCIPLE) 
For every Aj,..., An, 


Pr[U, Ad =  Pr[Ai— Y Pr[A;n Aj) +--+ + (-1)71 Pr[41 0---M An]. 
i=l 1<i<j<n 


Moreover, this is an alternating sum which means that if we take only the first k summands of the 
right hand side, then this upperbounds the left-hand side if k is odd, and lowerbounds it if k is 
even. 
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We sometimes use the following corollary of this claim: 


CLAIM A.4 
For every events Aj,..., An, 


PrlUjz14s] > XC Pr[A]- Y Pria;n Aj] 
i=l 1<i<j<n 


Random subsum principle. The following fact is used often in the book: 


CLAIM A.5 (THE RANDOM SUBSUM PRINCIPLE) 
For x,y € {0,1}", denote z © y = J; ziyi (mod 2) (that is, x © y is equal to 1 if the number of 
i’s such that x; = y; = 1 is odd and equal to 0 otherwise). Then for every y 4 0”, 


Pr rOy=l=i 
xe R{0,1}” Y l 2 


PROOF: Suppose that yj is nonzero. We can think of choosing x as follows: first choose all the 
coordinates of x other than the jt? and only choose the jt? coordinate last. After we choose all the 
coordinates of x other than the jt”, the value > imaj Liyi (mod 2) is fixed to be some c € {0,1}. 
Regardless of what c is, with probability 1/2 we choose x; = 0, in which case z © y = c and with 
probability 1/2 we choose x; = 1, in which case z © y = 1 — c. We see that in any case a [[ y will 
be equal to 1 with probability 1/2. Mi 


A.3.1 Random variables and expectations. 


A random variable is a mapping from a probability space to R. For example, if Q is as above, the 
set of all possible outcomes of n tosses of a fair coin, then we can denote by X the number of coins 
that came up heads. 

The expectation of a random variable X, denoted by E[X], is its weighted average. That is, 
E[X] = a piX (wi). The following simple claim follows from the definition: 


CLAIM A.6 (LINEARITY OF EXPECTATION) 
For X, Y random variables over a space (2, denote by X + Y the random variable that maps w to 
X(w) + Y (w). Then, 

E[X + Y] = E[X] + E[Y] 


This claims implies that the random variable X from the example above has expectation n/2. 
Indeed X = 57", X; where X; is equal to 1 if the it? coins came up heads and is equal to 0 
otherwise. But clearly, E[X;] = 1/2 for every i. 

For a real number a and a random variable X, we define aX to be the random variable mapping 
w to a: X(w). Note that Elo.X] = aE[X]. 
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A.3.2 The averaging argument 


We list various versions of the “averaging argument.” Sometimes we give two versions of the same 
result, one as a fact about numbers and one as a fact about probability spaces. 


LEMMA A.7 
If a1, a2,...,@n are some numbers whose average is c then some a; > c. 


LEMMA A.8 (“THE PROBABILISTIC METHOD” ) 
If X is a random variable which takes values from a finite set and E|X] = y then the event “X > u” 
has nonzero probability. 


LEMMA A.9 
Ifa¡,a2,...,an > 0 are numbers whose average is c then the fraction of a;’s that are greater than 
(resp., at least) kc is less than (resp, at most) 1/k. 


LEMMA A.10 (“MARKOV’S INEQUALITY” ) 
Any non-negative random variable X satisfies 


Pr (X > kE[X]) < 


A 


COROLLARY A.11 
If a1,a2,...,an € [0,1] are numbers whose average is 1 — y then at least 1 — y/y fraction of them 
are at least 1 — yy. 


Can we give any meaningful upperbound on Pr[X < c-E[X]] where c < 1? Yes, if X is bounded. 


LEMMA A.12 
If a1, a2,...,@,, are numbers in the interval [0,1] whose average is p then at least p/2 of the a;'s 
are at least as large as p/2. 


PROOF: Let y be the fraction of i’s such that a; > p/2. Then y+ (1 — y)p/2 must be at least p/2, 
so y > p/2. MW More generally, we have 


LEMMA A.13 
If X € [0,1] and E[X] = y then for any c < 1 we have 


EXAMPLE A.14 
Suppose you took a lot of exams, each scored from 1 to 100. If your average score was 90 then in 
at least half the exams you scored at least 80. 
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A.3.3 Conditional probability and independence 


If we already know that an event B happened, this reduces the space from Q to Q N B, where we 
need to scale the probabilities by 1/Pr[B] so they will sum up to one. Thus, the probability of 
an event A conditioned on an event B, denoted Pr[A|B], is equal to Pr[A N B]/Pr[B] (where we 
always assume that B has positive probability). 

We say that two events A, B are independent if Pr[AN B] = Pr[A] Pr[B]. Note that this implies 
that Pr[4|B] = Pr[A] and Pr[B|4] = Pr[B]. We say that a set of events Aj,..., An are mutually 
independent if for every subset S C [n], 


Pr[Mies Ai] = | [ Pri A (2) 
1€S 
We say that A1,..., An are k-wise independent if (2) holds for every S C [n] with |S] < k. 
We say that two random variables X,Y are independent if for every x,y € R, the events {X = x} 
and (Y = y) are independent. We generalize similarly the definition of mutual independence and 
k-wise independence to sets of random variables Xj,...,Xn. We have the following claim: 


CLAIM A.15 
If X1,...,Xp are mutually independent then 


PROOF: 
x 


bp £1: £n Pr| X; = zı and Xə = z2- -- and Xn = tn] = (by independence) 
Tison 
> ay. En Pr xy = z1] Pel xy = £n] = 


T1) Un 


n 


O 21 Pr[X, = 21) O z2 Pr[X2 = 29))- D a Xn = &]) = [TELS 


xi T2 i=1 


where the sums above are over all the possible real numbers that can be obtained by applying the 
random variables or their products to the finite set Q. W 


A.3.4 Deviation upperbounds 


Under various conditions, one can give upperbounds on the probability of a random variable “stray- 
ing too far” from its expectation. These upperbounds are usually derived by clever use of Markov’s 
inequality. 

The variance of a random variable X is defined to be Var[X] = E[(X —E(X))?]. Note that since 
it is the expectation of a non-negative random variable, Var[X] is always non-negative. Also, using 
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linearity of expectation, we can derive that Var[X] = E[X?] — (E[X])?. The standard deviation of 
a variable X is defined to be \/Var[X]. 
The first bound is Chebyshev’s inequality, useful when only the variance is known. 


LEMMA A.16 (CHEBYSHEV INEQUALITY) 
If X is a random variable with standard deviation o, then for every k > 0, 


Pr[|X — ELX]| > ko] < 1/k? 


PROOF: Apply Markov’s inequality to the random variable (X — ELX])?, noting that by definition 
of variance, E[(X — E[X])?] =0?. m 


Chebyshev’s inequality is often useful in the case that X is equal to >7;_, X; for pairwise 
independent random variables X1,..., Xn. This is because of the following claim, that is left as an 
exercise: 


CLAIM A.17 
If X1,...,Xp are pairwise independent then 


var Xi) = Y Var(X;) 
i=1 i=1 


The next inequality has many names, and is widely known in theoretical computer science as 
the Chernoff bound. It considers scenarios of the following type. Suppose we toss a fair coin n 
times. The expected number of heads is n/2. How tightly is this number concentrated? Should we 
be very surprised if after 1000 tosses we have 625 heads? The bound we present is slightly more 
general, since it concerns n different coin tosses of possibly different expectations (the expectation 
of a coin is the probability of obtaining “heads”; for a fair coin this is 1/2). These are sometimes 
known as Poisson trials. 


THEOREM A.18 (“CHERNOFF” BOUNDS) 
Let X 1, X2,..., Xn be mutually independent random variables over {0,1} (i.e., X; can be either 0 
or 1) and let u = X; E[X;]. Then for every 6 > 0, 


n eô H 
P X; > (1+ 6)p] < hs ; (3) 
i= a ne . 
Pr) sao s | (4) 


Often, what we use need is only the corollary that under the above conditions, for every c > 0 


e 


n 
De xX: =p 
a 


> as < 9-n/2 


A.3. PROBABILITY THEORY pA.9 (455) 


PROOF: Surprisingly, the Chernoff bound is also proved using the Markov inequality. We only 
prove the first inequality; a similar proof exists for the second. We introduce a positive dummy 
variable t, and observe that 


Elexp(tX)] = Elexp(t JS X)] = El] [ exp(tX] = [] Elexp (x9), (5) 


where exp(z) denotes e” and the last equality holds because the X; r.v.s are independent. Now, 
Efexp(tX;)] = (1 — pi) + pie’, 
therefore, 
] [ Elexp tX) = [ [E + pile’ — 1) < [Texp@ite’ - 1) 
a 2 2 A , (6) 
=expO) _ pie’ — 1)) = exp(u(e’ — 1)), 


2 


as 1+x< e”. Finally, apply Markov’s inequality to the random variable exp(tX), viz. 


E Elexp(t X] _ exp((et —1)u) 
Pr[X > (1+ 6)y] = Prlexp(tX) > exp(t(1 + 9)u)] < 0120 sion 


using lines (5) and (6) and the fact that t is positive. Since t is a dummy variable, we can choose 
any positive value we like for it. Simple calculus shows that the right hand side is minimized for 
t = In(1 + ô) and this leads to the theorem statement. Mi 


By the way, if all n coin tosses are fair (Heads has probability 1/2) then the the probability of 
seeing N heads where |N — n/2| > ayn is at most e-%/2. The chance of seeing at least 625 heads 
in 1000 tosses of an unbiased coin is less than 5.3 x 1077. 


A.3.5 Some other inequalities. 
Jensen’s inequality. 
The following inequality, generalizing the inequality E[X?] > ELX]?, is also often useful: 


CLAIM A.19 
We say that f : R — R is convex if for every p € [0,1] and z,y € R, f(px + (1 — p)y) < 
p- f(x)+(1—p)-f(y). Then, for every random variable X and convex function f, F(E[X]) < E[f(X)]. 


Approximating the binomial coefficient 
Of special interest is the Binomial random variable B, denoting the number of coins that come up 
“heads” when tossing n fair coins. For every k, Pr[B, = k] = 27” (}) where (7) = A denotes 


the number of size-k subsets of [n]. Clearly, (E) < n*, but sometimes we will need a better estimate 
for e and use the following approximation: 
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CLAIM A.20 
For every n,k <n, 


ny E n ne\k 
Gl te) 
The best approximation can be obtained via Stirling’s formula: 


LEMMA A.21 (STIRLING’S FORMULA) 


For every n, 
1 


n 1 n 
27n (=) el2n+1 <n! < v2rn (=) eT2n 
e e 
It can be proven by taking natural logarithms and approximating Inn! = In(1-2---n) = 
yoy, Ini by the integral JE ln zdr = nlnn —n + 1. It implies the following corollary: 


COROLLARY A.22 
For every n € N and a € [0,1], 


( É ) suton 52H 


an 2rma(1—a) 


where H(a) = alog(1/a) + (1 — a) log(1/(1 — a)) and the constants hidden in the O notation are 
independent of both n and a. 


More useful estimates. 


The following inequalities can be obtained via elementary calculus: 


x 
e For every x > 1, (1 +)" < L < (1 +) 


For every k, Xi =0 (5) 
e For every k>1, 2, n-* < O(1). 


e For every c,e > 0, 7721 asar < O(1). 


For every n, X; = Inn + O(1) 


A.4 Finite fields and groups 


A field is a set F that has an addition (+) and multiplication (-) operations that behave in the 
expected way: satisfy associative, commutative and distributive laws, have both additive and 
multiplicative inverses, and neutral elements 0 and 1 for addition and multiplication respectively. 
Familiar fields are the real numbers (R), the rational numbers (Q) and the complex numbers (C), 
but there are also finite fields. 

If q is a prime, then we denote by GF(q) the field consisting of the elements {0,...,q— 1} with 
addition and multiplication performed modulo q. For example, the numbers {0,...,6} yield a field 
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if addition and multiplication are performed modulo 7. We leave it to the reader to verify GF (q) is 
indeed a field for every prime q. The simplest example for such a field is the field GF(2) consisting 
of {0,1} where multiplication is the AND (A) operation and addition is the XOR operation. 

Every finite field F has a number £ such that for every x € F, +s +--- +x (£ times) is equal 
to the zero element of F (exercise). This number £ is called the characteristic of F. For every prime 
q, the characteristic of GF(q) is equal to q. 


A.4.1 Non-prime fields. 


One can see that if n is not prime, then the set {0,...,n — 1} with addition and multiplication 
modulo n is not a field, as there exist two non-zero elements x, y in this set such that -y=n=0 
(mod n). Nevertheless, there are finite fields of size n for non-prime n. Specifically, for every prime 
q, and k > 1, there exists a field of q! elements, which we denote by GF(q*). We will very rarely 
need to use such fields in this book, but still provide an outline of their construction below. 

For every prime q and k there exists an irreducible degree k polynomial P over the field GF(q) 
(P is irreducible if it cannot be expressed as the product of two polynomials P’, P” of lower degree). 
We then let GF(q") be the set of all k—1-degree polynomials over GF(q). Each such polynomial can 
be represented as a vector of its k coefficients. We perform both addition and multiplication modulo 
the polynomial P. Note that addition corresponds to standard vector addition of k-dimensional 
vectors over GF(q), and both addition and multiplication can be easily done in poly(n, log q) time 
(we can reduce a polynomial S modulo a polynomial P using a similar algorithm to long division 
of numbers). It turns out that no matter how we choose the irreducible polynomial P, we will get 
the same field, up to renaming of the elements. There is a deterministic poly(q, k)-time algorithm 
to obtain an irreducible polynomial of degree k over GF(q). There are also probabilistic algorithms 
(and deterministic algorithms whose analysis relies on unproven assumptions) that obtain such a 
polynomial in poly(log q, k) time. 

For us, the most important example of a finite field is GF(2*), which consists of the set {0,1}*, 
with addition being component-wise XOR, and multiplication being polynomial multiplication via 
some irreducible polynomial which we can fine in poly(k) time. In fact, we will mostly not even be 
interested in the multiplicative structure of GF(2*) and only use the addition operation (i.e., use 
it as the vector space GF(2)*, see below). 


A.4.2 Groups. 


A group is a set that only has a single operation, say x, that is associative and has an inverse. That 
is, (G,x) is a group if 


1. For every a,b,c € G , (axb) xc =ax (bxc) 


2. There exists a special element id € G such that a xid = a for every a € G, and for every 
a € G there exists b € G such that axb = bxa = id. 


If G is a finite group, it is known that for every a € G, axax--- xa (|G| times) is equal to 
the element zd. A group is called commutative or Abelian if its operation satisfies a xb = bx a for 
every a,n € G. For every number n > 2, the set ([0,...,n— 1} with the operation being addition 
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modulo n is an Abelian group. Also, the set {k : k € [n — 1], gcd(k,n) = 1} with the operation 
being multiplication modulo n is an Abelian group. 

If F is a field and k > 1, then the set of k-dimensional vectors of F (i.e., F*) together with 
the operation of componentwise addition, yields an Abelian group. As mentioned above, the most 
interesting special case for us is the group GF(2)* for some k. Note that in this group the identity 
element is the vector 0* and for every x € GF(2)*, x + x = 0%. This group is often referred to as 
the Boolean cube. 


A.5 Vector spaces and Hilbert spaces 


A.6 Polynomials 


We list some basic facts about univariate polynomials. 


THEOREM A.23 
A nonzero polynomial of degree d has at most d distinct roots. 


PROOF: Suppose p(x) = ae ceja? has d+1 distinct roots a1,...,@q41 in some field F. Then 


d 
Nas -ci = plaj) =0, 
i=0 


for j = 1,...,d + 1. This means that the system Ay = 0 with 


1 aj at ad 
1 ea a2 af 
= 2 2 2 
2 d 
l Gayi QJ e Qdp 


has a solution y = c. The matrix A is a Vandermonde matrix, and it can be shown that 


det A = J [o — 05), 
i>j 
which is nonzero for distinct a;. Hence rankA = d+ 1. The system Ay = 0 has therefore only a 
trivial solution — a contradiction toc 40. M 


THEOREM A.24 
For any set of pairs (aj, b1),...,(@a41, ba+1) there exists a unique polynomial g(x) of degree at most 
d such that g(ai) = bi for alli =1,2,...,d+1. 


PROOF: The requirements are satisfied by Lagrange Interpolating Polynomial: 
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If two polynomials g1(x), g2(x) satisfy the requirements then their difference p(x) = gi(x) — ga(x) is 
of degree at most d, and is zero for £ = aj,...,@q41. Thus, from the previous theorem, polynomial 
p(x) must be zero and polynomials gı (x), g(x) identical. W 


The following elementary result is usually attributed to Schwartz and Zippel in the computer 
science community, though it was certainly known earlier (see e.g. DeMillo and Lipton [?]). 


LEMMA A.25 
If a polynomial p(x1,%2,..., £m) over F = GF(q) is nonzero and has total degree at most d, then 


d 
Pr[p(a1..€m) # 0] > 1- =, 
q 
where the probability is over all choices of a1..am € F. 


PROOF: We use induction on m. If m = 1 the statement follows from Theorem A.23. Suppose the 
statement is true when the number of variables is at most m — 1. Then p can be written as 


d 
p(x1, £2, ai Lm) = S > vipi(ra, EEI , Tm), 
1=0 


where p; has total degree at most d — i. Since p is nonzero, at least one of p; is nonzero. Let k be 
the largest į such that p; is nonzero. Then by the inductive hypothesis, 
d—k 
Pr [pi(a2, a3,...,@m) Æ 0] 2 

@2,,3,..-,8m q 
Whenever p;(az,a3,...,dm) Æ 0, p(11,42,43,...,dp,) is a nonzero univariate polynomial of 
degree k, and hence becomes 0 only for at most k values of xı. Hence 
k d—k d 


Pr[p(a1-.am) 20] > (1-54 is, 
q q q 


and the induction is completed. M 
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